Skip to content

Commit

Permalink
Merge pull request #1183 from aryan9600/multi-cluster
Browse files Browse the repository at this point in the history 
Avoid setting owner refs if the service mesh/ingress is on a different cluster
  • Loading branch information
@stefanprodan
stefanprodan authored May 6, 2022
2 parents a8b4e9c + 8c690d1 commit 07bd356
Show file tree
Hide file tree
Showing 17 changed files with 225 additions and 157 deletions.
17 changes: 12 additions & 5 deletions cmd/flagger/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,15 +168,15 @@ func main() {
if kubeconfigServiceMesh == "" {
kubeconfigServiceMesh = kubeconfig
}
cfgHost, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigServiceMesh)
serviceMeshCfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigServiceMesh)
if err != nil {
logger.Fatalf("Error building host kubeconfig: %v", err)
}

cfgHost.QPS = float32(kubeconfigQPS)
cfgHost.Burst = kubeconfigBurst
serviceMeshCfg.QPS = float32(kubeconfigQPS)
serviceMeshCfg.Burst = kubeconfigBurst

meshClient, err := clientset.NewForConfig(cfgHost)
meshClient, err := clientset.NewForConfig(serviceMeshCfg)
if err != nil {
logger.Fatalf("Error building mesh clientset: %v", err)
}
Expand Down Expand Up @@ -212,7 +212,14 @@ func main() {
// start HTTP server
go server.ListenAndServe(port, 3*time.Second, logger, stopCh)

routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient)
setOwnerRefs := true
// Router shouldn't set OwnerRefs on resources that they create since the
// service mesh/ingress controller is in a different cluster.
if cfg.Host != serviceMeshCfg.Host {
setOwnerRefs = false
}

routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient, setOwnerRefs)

var configTracker canary.Tracker
if enableConfigTracking {
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/scheduler_daemonset_fixture_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func newDaemonSetFixture(c *flaggerv1.Canary) daemonSetFixture {
}

// init router
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient)
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true)

// init observer
observerFactory, _ := observers.NewFactory(testMetricsServerURL)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/scheduler_deployment_fixture_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture {
}

// init router
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient)
rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true)

// init observer
observerFactory, _ := observers.NewFactory(testMetricsServerURL)
Expand Down
33 changes: 19 additions & 14 deletions pkg/router/appmesh.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type AppMeshRouter struct {
appmeshClient clientset.Interface
flaggerClient clientset.Interface
logger *zap.SugaredLogger
setOwnerRefs bool
}

// Reconcile creates or updates App Mesh virtual nodes and virtual services
Expand Down Expand Up @@ -161,16 +162,18 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str
Namespace: canary.Namespace,
Labels: metadata.Labels,
Annotations: filterMetadata(metadata.Annotations),
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vnSpec,
}
if ar.setOwnerRefs {
virtualnode.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}
_, err = ar.appmeshClient.AppmeshV1beta1().VirtualNodes(canary.Namespace).Create(context.TODO(), virtualnode, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("VirtualNode %s.%s create error %w", name, canary.Namespace, err)
Expand Down Expand Up @@ -314,16 +317,18 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: canary.Namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vsSpec,
}
if ar.setOwnerRefs {
virtualService.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

// set App Mesh Gateway annotation on primary virtual service
if canaryWeight == 0 {
Expand Down
49 changes: 28 additions & 21 deletions pkg/router/appmesh_v1beta2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type AppMeshv1beta2Router struct {
flaggerClient clientset.Interface
logger *zap.SugaredLogger
labelSelector string
setOwnerRefs bool
}

// Reconcile creates or updates App Mesh virtual nodes and virtual services
Expand Down Expand Up @@ -175,16 +176,18 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualNode(canary *flaggerv1.Canary, n
Namespace: canary.Namespace,
Labels: metadata.Labels,
Annotations: filterMetadata(metadata.Annotations),
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vnSpec,
}
if ar.setOwnerRefs {
virtualnode.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}
_, err = ar.appmeshClient.AppmeshV1beta2().VirtualNodes(canary.Namespace).Create(context.TODO(), virtualnode, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("VirtualNode %s.%s create error %w", name, canary.Namespace, err)
Expand Down Expand Up @@ -341,16 +344,18 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: canary.Namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: vrSpec,
}
if ar.setOwnerRefs {
virtualRouter.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

_, err = ar.appmeshClient.AppmeshV1beta2().VirtualRouters(canary.Namespace).Create(context.TODO(), virtualRouter, metav1.CreateOptions{})
if err != nil {
Expand All @@ -363,13 +368,6 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: canary.Namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: appmeshv1.VirtualServiceSpec{
Provider: &appmeshv1.VirtualServiceProvider{
Expand All @@ -381,6 +379,15 @@ func (ar *AppMeshv1beta2Router) reconcileVirtualRouter(canary *flaggerv1.Canary,
},
},
}
if ar.setOwnerRefs {
virtualService.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

// set App Mesh Gateway annotation on primary virtual service
if canaryWeight == 0 {
Expand Down
17 changes: 10 additions & 7 deletions pkg/router/contour.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type ContourRouter struct {
flaggerClient clientset.Interface
logger *zap.SugaredLogger
ingressClass string
setOwnerRefs bool
}

// Reconcile creates or updates the HTTP proxy
Expand Down Expand Up @@ -169,20 +170,22 @@ func (cr *ContourRouter) Reconcile(canary *flaggerv1.Canary) error {
Namespace: canary.Namespace,
Labels: metadata.Labels,
Annotations: filterMetadata(metadata.Annotations),
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Spec: newSpec,
Status: contourv1.HTTPProxyStatus{
CurrentStatus: "valid",
Description: "valid HTTPProxy",
},
}
if cr.setOwnerRefs {
proxy.OwnerReferences = []metav1.OwnerReference{
*metav1.NewControllerRef(canary, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
}
}

if cr.ingressClass != "" {
proxy.Annotations = map[string]string{
Expand Down
24 changes: 21 additions & 3 deletions pkg/router/factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,16 @@ type Factory struct {
ingressAnnotationsPrefix string
ingressClass string
logger *zap.SugaredLogger
setOwnerRefs bool
}

func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
flaggerClient clientset.Interface,
ingressAnnotationsPrefix string,
ingressClass string,
logger *zap.SugaredLogger,
meshClient clientset.Interface) *Factory {
meshClient clientset.Interface,
setOwnerRefs bool) *Factory {
return &Factory{
kubeConfig: kubeConfig,
meshClient: meshClient,
Expand All @@ -51,6 +53,7 @@ func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
ingressAnnotationsPrefix: ingressAnnotationsPrefix,
ingressClass: ingressClass,
logger: logger,
setOwnerRefs: setOwnerRefs,
}
}

Expand Down Expand Up @@ -81,13 +84,15 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
appmeshClient: factory.meshClient,
labelSelector: labelSelector,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.AppMeshProvider:
return &AppMeshRouter{
logger: factory.logger,
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
appmeshClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.LinkerdProvider:
return &SmiRouter{
Expand All @@ -96,13 +101,15 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: flaggerv1.LinkerdProvider,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.IstioProvider:
return &IstioRouter{
logger: factory.logger,
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
istioClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha1"):
mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha1:")
Expand All @@ -112,6 +119,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: mesh,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha2"):
mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha2:")
Expand All @@ -121,6 +129,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: mesh,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.SMIProvider+":v1alpha3"):
mesh := strings.TrimPrefix(provider, flaggerv1.SMIProvider+":v1alpha3:")
Expand All @@ -130,6 +139,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: mesh,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.ContourProvider:
return &ContourRouter{
Expand All @@ -138,29 +148,34 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
contourClient: factory.meshClient,
ingressClass: factory.ingressClass,
setOwnerRefs: factory.setOwnerRefs,
}
case strings.HasPrefix(provider, flaggerv1.GlooProvider):
return &GlooRouter{
logger: factory.logger,
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
glooClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.NGINXProvider:
return &IngressRouter{
logger: factory.logger,
kubeClient: factory.kubeClient,
annotationsPrefix: factory.ingressAnnotationsPrefix,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.SkipperProvider:
return &SkipperRouter{
logger: factory.logger,
kubeClient: factory.kubeClient,
logger: factory.logger,
kubeClient: factory.kubeClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.TraefikProvider:
return &TraefikRouter{
logger: factory.logger,
traefikClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.OsmProvider:
return &Smiv1alpha2Router{
Expand All @@ -169,6 +184,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
kubeClient: factory.kubeClient,
smiClient: factory.meshClient,
targetMesh: flaggerv1.OsmProvider,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.KumaProvider:
return &KumaRouter{
Expand All @@ -182,6 +198,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
logger: factory.logger,
kubeClient: factory.kubeClient,
gatewayAPIClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
case provider == flaggerv1.KubernetesProvider:
return &NopRouter{}
Expand All @@ -191,6 +208,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
flaggerClient: factory.flaggerClient,
kubeClient: factory.kubeClient,
istioClient: factory.meshClient,
setOwnerRefs: factory.setOwnerRefs,
}
}
}
Loading

0 comments on commit 07bd356

Please sign in to comment.