From 48c981f2ed2369c6f698949cd13b185cc50256d1 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 25 May 2021 11:14:45 +0300 Subject: [PATCH 01/19] chore(*) add ZoneIngress resource Signed-off-by: Nikolay Nikolaev --- api/mesh/v1alpha1/zone_ingress.pb.go | 313 ++++++++++++++++++ api/mesh/v1alpha1/zone_ingress.proto | 41 +++ pkg/core/resources/apis/mesh/zone_ingress.go | 90 +++++ .../apis/mesh/zone_ingress_validator.go | 48 +++ .../apis/mesh/zone_ingress_validator_test.go | 147 ++++++++ pkg/core/xds/types.go | 13 +- 6 files changed, 646 insertions(+), 6 deletions(-) create mode 100644 api/mesh/v1alpha1/zone_ingress.pb.go create mode 100644 api/mesh/v1alpha1/zone_ingress.proto create mode 100644 pkg/core/resources/apis/mesh/zone_ingress.go create mode 100644 pkg/core/resources/apis/mesh/zone_ingress_validator.go create mode 100644 pkg/core/resources/apis/mesh/zone_ingress_validator_test.go diff --git a/api/mesh/v1alpha1/zone_ingress.pb.go b/api/mesh/v1alpha1/zone_ingress.pb.go new file mode 100644 index 000000000000..1c6788659eb1 --- /dev/null +++ b/api/mesh/v1alpha1/zone_ingress.pb.go @@ -0,0 +1,313 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.23.0 +// protoc v3.14.0 +// source: mesh/v1alpha1/zone_ingress.proto + +package v1alpha1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + proto "github.com/golang/protobuf/proto" + _ "github.com/golang/protobuf/ptypes/duration" + _ "github.com/golang/protobuf/ptypes/wrappers" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// ZoneIngress allows us to configure dataplane in the Ingress mode. In this +// mode, dataplane has only inbound interfaces. Every inbound interface matches +// with services that reside in that cluster. +type ZoneIngress struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Address on which inbound listener will be exposed + Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` + // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible to + // other Kuma clusters. + AdvertisedAddress string `protobuf:"bytes,2,opt,name=advertisedAddress,proto3" json:"advertisedAddress,omitempty"` + // Port of the inbound interface that will forward requests to the service. + Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + // AdvertisedPort defines port on which ZoneIngress is accessible to other Kuma + // clusters. + AdvertisedPort uint32 `protobuf:"varint,4,opt,name=advertisedPort,proto3" json:"advertisedPort,omitempty"` + AvailableServices []*ZoneIngress_AvailableService `protobuf:"bytes,5,rep,name=availableServices,proto3" json:"availableServices,omitempty"` +} + +func (x *ZoneIngress) Reset() { + *x = ZoneIngress{} + if protoimpl.UnsafeEnabled { + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ZoneIngress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ZoneIngress) ProtoMessage() {} + +func (x *ZoneIngress) ProtoReflect() protoreflect.Message { + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ZoneIngress.ProtoReflect.Descriptor instead. +func (*ZoneIngress) Descriptor() ([]byte, []int) { + return file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP(), []int{0} +} + +func (x *ZoneIngress) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *ZoneIngress) GetAdvertisedAddress() string { + if x != nil { + return x.AdvertisedAddress + } + return "" +} + +func (x *ZoneIngress) GetPort() uint32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ZoneIngress) GetAdvertisedPort() uint32 { + if x != nil { + return x.AdvertisedPort + } + return 0 +} + +func (x *ZoneIngress) GetAvailableServices() []*ZoneIngress_AvailableService { + if x != nil { + return x.AvailableServices + } + return nil +} + +// AvailableService contains tags that represent unique subset of +// endpoints +type ZoneIngress_AvailableService struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // tags of the service + Tags map[string]string `protobuf:"bytes,1,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // number of instances available for given tags + Instances uint32 `protobuf:"varint,2,opt,name=instances,proto3" json:"instances,omitempty"` + // mesh of the instances available for given tags + Mesh string `protobuf:"bytes,3,opt,name=mesh,proto3" json:"mesh,omitempty"` +} + +func (x *ZoneIngress_AvailableService) Reset() { + *x = ZoneIngress_AvailableService{} + if protoimpl.UnsafeEnabled { + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ZoneIngress_AvailableService) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ZoneIngress_AvailableService) ProtoMessage() {} + +func (x *ZoneIngress_AvailableService) ProtoReflect() protoreflect.Message { + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ZoneIngress_AvailableService.ProtoReflect.Descriptor instead. +func (*ZoneIngress_AvailableService) Descriptor() ([]byte, []int) { + return file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *ZoneIngress_AvailableService) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} + +func (x *ZoneIngress_AvailableService) GetInstances() uint32 { + if x != nil { + return x.Instances + } + return 0 +} + +func (x *ZoneIngress_AvailableService) GetMesh() string { + if x != nil { + return x.Mesh + } + return "" +} + +var File_mesh_v1alpha1_zone_ingress_proto protoreflect.FileDescriptor + +var file_mesh_v1alpha1_zone_ingress_proto_rawDesc = []byte{ + 0x0a, 0x20, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, + 0x7a, 0x6f, 0x6e, 0x65, 0x5f, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x12, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x1a, 0x1b, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc1, 0x03, 0x0a, + 0x0b, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, + 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x64, 0x76, 0x65, + 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, + 0x12, 0x5e, 0x0a, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, + 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, + 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x11, 0x61, + 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x1a, 0xcd, 0x01, 0x0a, 0x10, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, + 0x63, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, + 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, + 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_mesh_v1alpha1_zone_ingress_proto_rawDescOnce sync.Once + file_mesh_v1alpha1_zone_ingress_proto_rawDescData = file_mesh_v1alpha1_zone_ingress_proto_rawDesc +) + +func file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP() []byte { + file_mesh_v1alpha1_zone_ingress_proto_rawDescOnce.Do(func() { + file_mesh_v1alpha1_zone_ingress_proto_rawDescData = protoimpl.X.CompressGZIP(file_mesh_v1alpha1_zone_ingress_proto_rawDescData) + }) + return file_mesh_v1alpha1_zone_ingress_proto_rawDescData +} + +var file_mesh_v1alpha1_zone_ingress_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_mesh_v1alpha1_zone_ingress_proto_goTypes = []interface{}{ + (*ZoneIngress)(nil), // 0: kuma.mesh.v1alpha1.ZoneIngress + (*ZoneIngress_AvailableService)(nil), // 1: kuma.mesh.v1alpha1.ZoneIngress.AvailableService + nil, // 2: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry +} +var file_mesh_v1alpha1_zone_ingress_proto_depIdxs = []int32{ + 1, // 0: kuma.mesh.v1alpha1.ZoneIngress.availableServices:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService + 2, // 1: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.tags:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_mesh_v1alpha1_zone_ingress_proto_init() } +func file_mesh_v1alpha1_zone_ingress_proto_init() { + if File_mesh_v1alpha1_zone_ingress_proto != nil { + return + } + file_mesh_v1alpha1_metrics_proto_init() + if !protoimpl.UnsafeEnabled { + file_mesh_v1alpha1_zone_ingress_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ZoneIngress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ZoneIngress_AvailableService); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_mesh_v1alpha1_zone_ingress_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_mesh_v1alpha1_zone_ingress_proto_goTypes, + DependencyIndexes: file_mesh_v1alpha1_zone_ingress_proto_depIdxs, + MessageInfos: file_mesh_v1alpha1_zone_ingress_proto_msgTypes, + }.Build() + File_mesh_v1alpha1_zone_ingress_proto = out.File + file_mesh_v1alpha1_zone_ingress_proto_rawDesc = nil + file_mesh_v1alpha1_zone_ingress_proto_goTypes = nil + file_mesh_v1alpha1_zone_ingress_proto_depIdxs = nil +} diff --git a/api/mesh/v1alpha1/zone_ingress.proto b/api/mesh/v1alpha1/zone_ingress.proto new file mode 100644 index 000000000000..fdb7de95ec74 --- /dev/null +++ b/api/mesh/v1alpha1/zone_ingress.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package kuma.mesh.v1alpha1; + +option go_package = "github.com/kumahq/kuma/api/mesh/v1alpha1"; + +import "mesh/v1alpha1/metrics.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; +import "validate/validate.proto"; + +// ZoneIngress allows us to configure dataplane in the Ingress mode. In this +// mode, dataplane has only inbound interfaces. Every inbound interface matches +// with services that reside in that cluster. +message ZoneIngress { + // Address on which inbound listener will be exposed + string address = 1; + + // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible + // to other Kuma clusters. + string advertisedAddress = 2; + + // Port of the inbound interface that will forward requests to the service. + uint32 port = 3; + + // AdvertisedPort defines port on which ZoneIngress is accessible to other + // Kuma clusters. + uint32 advertisedPort = 4; + + // AvailableService contains tags that represent unique subset of + // endpoints + message AvailableService { + // tags of the service + map tags = 1; + // number of instances available for given tags + uint32 instances = 2; + // mesh of the instances available for given tags + string mesh = 3; + } + repeated AvailableService availableServices = 5; +} diff --git a/pkg/core/resources/apis/mesh/zone_ingress.go b/pkg/core/resources/apis/mesh/zone_ingress.go new file mode 100644 index 000000000000..8be2c59725d4 --- /dev/null +++ b/pkg/core/resources/apis/mesh/zone_ingress.go @@ -0,0 +1,90 @@ +package mesh + +import ( + "errors" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core/resources/model" + "github.com/kumahq/kuma/pkg/core/resources/registry" +) + +const ( + ZoneIngressType model.ResourceType = "ZoneIngress" +) + +var _ model.Resource = &ZoneIngressResource{} + +type ZoneIngressResource struct { + Meta model.ResourceMeta + Spec *mesh_proto.ZoneIngress +} + +func NewZoneIngressResource() *ZoneIngressResource { + return &ZoneIngressResource{ + Spec: &mesh_proto.ZoneIngress{}, + } +} + +func (t *ZoneIngressResource) GetType() model.ResourceType { + return ZoneIngressType +} +func (t *ZoneIngressResource) GetMeta() model.ResourceMeta { + return t.Meta +} +func (t *ZoneIngressResource) SetMeta(m model.ResourceMeta) { + t.Meta = m +} +func (t *ZoneIngressResource) GetSpec() model.ResourceSpec { + return t.Spec +} +func (t *ZoneIngressResource) SetSpec(spec model.ResourceSpec) error { + zoneIngress, ok := spec.(*mesh_proto.ZoneIngress) + if !ok { + return errors.New("invalid type of spec") + } else { + t.Spec = zoneIngress + return nil + } +} + +func (t *ZoneIngressResource) Scope() model.ResourceScope { + return model.ScopeGlobal +} + +var _ model.ResourceList = &DataplaneResourceList{} + +type ZoneIngressResourceList struct { + Items []*ZoneIngressResource + Pagination model.Pagination +} + +func (l *ZoneIngressResourceList) GetItems() []model.Resource { + res := make([]model.Resource, len(l.Items)) + for i, elem := range l.Items { + res[i] = elem + } + return res +} + +func (l *ZoneIngressResourceList) GetItemType() model.ResourceType { + return ZoneIngressType +} +func (l *ZoneIngressResourceList) NewItem() model.Resource { + return NewZoneIngressResource() +} +func (l *ZoneIngressResourceList) AddItem(r model.Resource) error { + if trr, ok := r.(*ZoneIngressResource); ok { + l.Items = append(l.Items, trr) + return nil + } else { + return model.ErrorInvalidItemType((*ZoneIngressResource)(nil), r) + } +} +func (l *ZoneIngressResourceList) GetPagination() *model.Pagination { + return &l.Pagination +} + +func init() { + registry.RegisterType(NewZoneIngressResource()) + registry.RegistryListType(&ZoneIngressResourceList{}) +} diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator.go b/pkg/core/resources/apis/mesh/zone_ingress_validator.go new file mode 100644 index 000000000000..5b95d2a35190 --- /dev/null +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator.go @@ -0,0 +1,48 @@ +package mesh + +import ( + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core/validators" +) + +func (d *ZoneIngressResource) Validate() error { + var err validators.ValidationError + err.Add(validateZoneIngress(validators.RootedAt("ingress"), d.Spec)) + return err.OrNil() +} + +func validateZoneIngress(path validators.PathBuilder, ingress *mesh_proto.ZoneIngress) validators.ValidationError { + if ingress == nil { + return validators.ValidationError{} + } + var err validators.ValidationError + if ingress.GetAdvertisedAddress() == "" && ingress.GetAdvertisedPort() != 0 { + err.AddViolationAt(path.Field("advertisedAddress"), `has to be defined with advertisedPort`) + } + if ingress.GetAdvertisedPort() == 0 && ingress.GetAdvertisedAddress() != "" { + err.AddViolationAt(path.Field("advertisedPort"), `has to be defined with advertisedAddress`) + } + if ingress.GetAddress() != "" { + err.Add(validateAddress(path.Field("address"), ingress.GetAddress())) + } + if ingress.GetAdvertisedAddress() != "" { + err.Add(validateAddress(path.Field("advertisedAddress"), ingress.GetAdvertisedAddress())) + } + if ingress.GetPort() == 0 { + err.AddViolationAt(path.Field("port"), `port has to be defined`) + } + if ingress.GetPort() > 65535 { + err.AddViolationAt(path.Field("port"), `port has to be in range of [1, 65535]`) + } + if ingress.GetAdvertisedPort() > 65535 { + err.AddViolationAt(path.Field("advertisedPort"), `port has to be in range of [1, 65535]`) + } + for i, ingressInterface := range ingress.GetAvailableServices() { + p := path.Field("availableService").Index(i) + if _, ok := ingressInterface.Tags[mesh_proto.ServiceTag]; !ok { + err.AddViolationAt(p.Field("tags").Key(mesh_proto.ServiceTag), "cannot be empty") + } + err.AddErrorAt(p.Field("tags"), validateTags(ingressInterface.GetTags())) + } + return err +} diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go new file mode 100644 index 000000000000..73f02e5e9f97 --- /dev/null +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go @@ -0,0 +1,147 @@ +package mesh_test + +import ( + "github.com/ghodss/yaml" + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + util_proto "github.com/kumahq/kuma/pkg/util/proto" +) + +var _ = Describe("Dataplane", func() { + + DescribeTable("should pass validation", + func(dpYAML string) { + // given + zoneingress := core_mesh.NewZoneIngressResource() + + // when + err := util_proto.FromYAML([]byte(dpYAML), zoneingress.Spec) + + // then + Expect(err).ToNot(HaveOccurred()) + + // when + err = zoneingress.Validate() + + // then + Expect(err).ToNot(HaveOccurred()) + }, + Entry("with advertised address and port", ` + type: ZoneIngress + name: zi-1 + address: 192.168.0.1 + advertisedAddress: 10.0.0.1 + port: 10001 + advertisedPort: 1234 + availableServices: + - tags: + kuma.io/service: backend + version: "1" + region: us + - tags: + kuma.io/service: web + version: v2 + region: eu`, + ), + Entry("with advertised ipv6 address and port", ` + type: ZoneIngress + name: zi-1 + address: 192.168.0.1 + advertisedAddress: ::ffff:0a00:0001 + port: 10001 + advertisedPort: 1234 + availableServices: + - tags: + kuma.io/service: backend + version: "1" + region: us + - tags: + kuma.io/service: web + version: v2 + region: eu`, + ), + // no advertised address and port is valid because we may be waiting for Kubernetes to reconcile it + Entry("without advertised address and port", ` + type: ZoneIngress + name: zi-1 + address: 192.168.0.1 + port: 10001 + availableServices: []`, + ), + ) + + type testCase struct { + dataplane string + expected string + } + DescribeTable("should validate all fields and return as much individual errors as possible", + func(given testCase) { + // setup + zoneingress := core_mesh.NewZoneIngressResource() + + // when + err := util_proto.FromYAML([]byte(given.dataplane), zoneingress.Spec) + // then + Expect(err).ToNot(HaveOccurred()) + + // when + verr := zoneingress.Validate() + // and + actual, err := yaml.Marshal(verr) + + // then + Expect(err).ToNot(HaveOccurred()) + // and + Expect(actual).To(MatchYAML(given.expected)) + }, + Entry("no port defined", testCase{ + dataplane: ` + type: ZoneIngress + name: zi-1 + address: 192.168.0.1 + advertisedAddress: 10.0.0.1 + advertisedPort: 1234 + availableServices: + - tags: + kuma.io/service: backend + version: "1" + region: us + - tags: + kuma.io/service: web + version: v2 + region: eu`, + expected: ` + violations: + - field: ingress.port + message: port has to be defined`, + }), + Entry("invalid advertised address and port", testCase{ + dataplane: ` + type: ZoneIngress + name: zi-1 + address: 192.168.0.1 + advertisedAddress: "!@#" + port: 10001 + advertisedPort: 100000 + availableServices: + - tags: + kuma.io/service: backend + version: "1" + region: us + - tags: + kuma.io/service: web + version: v2 + region: eu`, + expected: ` + violations: + - field: ingress.advertisedAddress.address + message: address has to be valid IP address or domain name + - field: ingress.advertisedPort + message: port has to be in range of [1, 65535]`, + }), + ) + +}) diff --git a/pkg/core/xds/types.go b/pkg/core/xds/types.go index ec6de66621a2..a2a49c51cbfa 100644 --- a/pkg/core/xds/types.go +++ b/pkg/core/xds/types.go @@ -106,12 +106,13 @@ const ( ) type Proxy struct { - Id ProxyId - APIVersion envoy_common.APIVersion // todo(jakubdyszkiewicz) consider moving APIVersion here. pkg/core should not depend on pkg/xds. It should be other way around. - Dataplane *mesh_core.DataplaneResource - Metadata *DataplaneMetadata - Routing Routing - Policies MatchedPolicies + Id ProxyId + APIVersion envoy_common.APIVersion // todo(jakubdyszkiewicz) consider moving APIVersion here. pkg/core should not depend on pkg/xds. It should be other way around. + Dataplane *mesh_core.DataplaneResource + ZoneIngress *mesh_core.ZoneIngressResource + Metadata *DataplaneMetadata + Routing Routing + Policies MatchedPolicies } type Routing struct { From e7f4cefd8c694619f4a55966615c326bb7104c0c Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Thu, 10 Jun 2021 18:37:14 +0700 Subject: [PATCH 02/19] chore(kuma-cp) fix things here and there Signed-off-by: Ilya Lobkov --- api/mesh/v1alpha1/zone_ingress.pb.go | 72 +++++++----- api/mesh/v1alpha1/zone_ingress.proto | 9 +- app/kuma-dp/cmd/dataplane.go | 18 ++- app/kuma-dp/cmd/run.go | 34 ++++-- .../pkg/dataplane/envoy/remote_bootstrap.go | 11 +- app/kumactl/pkg/tokens/client_test.go | 17 ++- .../charts/kuma/crds/zoneingresses.yaml | 27 +++++ .../charts/kuma/templates/cp-rbac.yaml | 1 + .../kuma/templates/ingress-deployment.yaml | 4 +- pkg/api-server/definitions/all.go | 1 + pkg/api-server/definitions/zone-ingress.go | 17 +++ pkg/api-server/server.go | 8 +- pkg/config/app/kuma-dp/config.go | 4 +- pkg/core/bootstrap/bootstrap.go | 5 +- pkg/core/resources/apis/mesh/zone_ingress.go | 20 ++-- .../apis/mesh/zone_ingress_helpers.go | 69 +++++++++++ .../apis/mesh/zone_ingress_validator.go | 4 +- pkg/core/xds/metadata.go | 31 +++-- pkg/core/xds/types.go | 52 +++++++-- pkg/core/xds/types_test.go | 16 ++- pkg/defaults/components.go | 59 ++++++++-- pkg/defaults/components_test.go | 63 +++++++++- pkg/defaults/mesh.go | 6 +- pkg/defaults/signing_key.go | 29 +++++ pkg/dns/outbound.go | 22 +++- pkg/dns/outbound_test.go | 6 +- pkg/dns/vips_allocator.go | 19 ++- pkg/hds/authn/callbacks_test.go | 4 + pkg/kds/context/context.go | 15 ++- pkg/kds/global/components.go | 40 ++++--- pkg/kds/remote/components.go | 18 ++- pkg/kds/types.go | 2 + .../k8s/xds/hooks/api_server_bypass.go | 4 + .../native/api/v1alpha1/zone_ingress_types.go | 47 ++++++++ .../v1alpha1/zone_ingress_types_deepcopy.go | 63 ++++++++++ .../v1alpha1/zone_ingress_types_helpers.go | 61 ++++++++++ .../crd/bases/kuma.io_zoneingresses.yaml | 27 +++++ .../k8s/controllers/configmap_controller.go | 40 +++++++ .../k8s/controllers/ingress_converter.go | 19 ++- .../k8s/controllers/outbound_converter.go | 15 ++- .../runtime/k8s/controllers/pod_controller.go | 34 +++++- .../runtime/k8s/controllers/pod_converter.go | 19 +-- .../k8s/controllers/pod_converter_test.go | 23 ++-- .../testdata/ingress/01.dataplane.yaml | 17 +-- .../testdata/ingress/02.dataplane.yaml | 17 +-- .../testdata/ingress/03.dataplane.yaml | 13 +-- .../testdata/ingress/04.dataplane.yaml | 17 +-- .../testdata/ingress/05.dataplane.yaml | 17 +-- .../testdata/ingress/06.dataplane.yaml | 17 +-- .../runtime/universal/outbound/outbound.go | 29 +++-- pkg/sds/server/v2/reconciler.go | 12 +- pkg/sds/server/v2/server.go | 8 +- pkg/sds/server/v3/reconciler.go | 12 +- pkg/sds/server/v3/server.go | 8 +- pkg/tokens/builtin/components.go | 7 ++ .../server/types/zoneingress_token_request.go | 5 + pkg/tokens/builtin/server/webservice.go | 53 +++++++-- pkg/tokens/builtin/server/webservice_test.go | 17 ++- pkg/tokens/builtin/zoneingress/issuer.go | 91 +++++++++++++++ pkg/tokens/builtin/zoneingress/signing_key.go | 62 ++++++++++ pkg/xds/auth/callbacks.go | 54 ++++++++- pkg/xds/auth/callbacks_test.go | 4 + pkg/xds/auth/components/components.go | 6 +- pkg/xds/auth/interfaces.go | 1 + pkg/xds/auth/k8s/authenticator.go | 33 ++++++ pkg/xds/auth/universal/auth_test.go | 7 +- pkg/xds/auth/universal/authenticator.go | 22 +++- pkg/xds/auth/universal/noop_authenticator.go | 4 + pkg/xds/bootstrap/generator.go | 108 +++++++++++++----- pkg/xds/bootstrap/types/bootstrap_request.go | 1 + pkg/xds/cache/cla/cache.go | 6 +- pkg/xds/cache/mesh/snapshot.go | 7 ++ pkg/xds/generator/admin_proxy_generator.go | 56 +++++---- pkg/xds/generator/ingress_generator.go | 17 ++- pkg/xds/generator/ingress_generator_test.go | 21 ++-- pkg/xds/ingress/dataplane.go | 69 ++++++++++- pkg/xds/ingress/dataplane_test.go | 14 ++- pkg/xds/ingress/router.go | 4 +- pkg/xds/ingress/router_test.go | 13 ++- .../server/callbacks/dataplane_lifecycle.go | 65 +++++++++-- .../callbacks/dataplane_status_tracker.go | 4 +- .../callbacks/dataplane_sync_tracker.go | 5 +- .../callbacks/dataplane_sync_tracker_test.go | 6 +- pkg/xds/server/components.go | 4 +- pkg/xds/sync/dataplane_proxy_builder.go | 9 +- pkg/xds/sync/dataplane_watchdog.go | 49 ++++---- pkg/xds/sync/dataplane_watchdog_factory.go | 5 +- pkg/xds/sync/ingress_proxy_builder.go | 88 +++++++++++--- pkg/xds/sync/interfaces.go | 3 +- pkg/xds/topology/outbound.go | 52 ++++++++- pkg/xds/topology/outbound_test.go | 4 +- test/e2e/deploy/kuma_deploy_universal.go | 6 +- .../healthcheck/hybrid/healthcheck_hybrid.go | 4 +- test/e2e/hybrid/kuma_hybrid.go | 6 +- test/e2e/hybrid/kuma_hybrid_kube_global.go | 4 +- .../hybrid/traffic_permission_hybrid.go | 4 +- .../universal_multizone/traffic_route.go | 6 +- test/framework/interface.go | 1 + test/framework/k8s_controlplane.go | 14 +++ test/framework/setup.go | 28 ++++- test/framework/universal_app.go | 16 ++- test/framework/universal_cluster.go | 9 +- test/framework/universal_clusters.go | 6 +- test/framework/universal_controlplane.go | 17 +++ 104 files changed, 1853 insertions(+), 466 deletions(-) create mode 100644 deployments/charts/kuma/crds/zoneingresses.yaml create mode 100644 pkg/api-server/definitions/zone-ingress.go create mode 100644 pkg/core/resources/apis/mesh/zone_ingress_helpers.go create mode 100644 pkg/defaults/signing_key.go create mode 100644 pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types.go create mode 100644 pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_deepcopy.go create mode 100644 pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_helpers.go create mode 100644 pkg/plugins/resources/k8s/native/config/crd/bases/kuma.io_zoneingresses.yaml create mode 100644 pkg/tokens/builtin/server/types/zoneingress_token_request.go create mode 100644 pkg/tokens/builtin/zoneingress/issuer.go create mode 100644 pkg/tokens/builtin/zoneingress/signing_key.go diff --git a/api/mesh/v1alpha1/zone_ingress.pb.go b/api/mesh/v1alpha1/zone_ingress.pb.go index 1c6788659eb1..1d32a4af3f11 100644 --- a/api/mesh/v1alpha1/zone_ingress.pb.go +++ b/api/mesh/v1alpha1/zone_ingress.pb.go @@ -38,14 +38,18 @@ type ZoneIngress struct { // Address on which inbound listener will be exposed Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` - // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible to - // other Kuma clusters. + // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible + // to other Kuma clusters. AdvertisedAddress string `protobuf:"bytes,2,opt,name=advertisedAddress,proto3" json:"advertisedAddress,omitempty"` // Port of the inbound interface that will forward requests to the service. Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` - // AdvertisedPort defines port on which ZoneIngress is accessible to other Kuma - // clusters. - AdvertisedPort uint32 `protobuf:"varint,4,opt,name=advertisedPort,proto3" json:"advertisedPort,omitempty"` + // AdvertisedPort defines port on which ZoneIngress is accessible to other + // Kuma clusters. + AdvertisedPort uint32 `protobuf:"varint,4,opt,name=advertisedPort,proto3" json:"advertisedPort,omitempty"` + // Zone field contains Zone name where ingress is serving, field will be automatically set by Global Kuma CP + Zone string `protobuf:"bytes,6,opt,name=zone,proto3" json:"zone,omitempty"` + // AvailableService contains tags that represent unique subset of + // endpoints AvailableServices []*ZoneIngress_AvailableService `protobuf:"bytes,5,rep,name=availableServices,proto3" json:"availableServices,omitempty"` } @@ -109,6 +113,13 @@ func (x *ZoneIngress) GetAdvertisedPort() uint32 { return 0 } +func (x *ZoneIngress) GetZone() string { + if x != nil { + return x.Zone + } + return "" +} + func (x *ZoneIngress) GetAvailableServices() []*ZoneIngress_AvailableService { if x != nil { return x.AvailableServices @@ -116,8 +127,6 @@ func (x *ZoneIngress) GetAvailableServices() []*ZoneIngress_AvailableService { return nil } -// AvailableService contains tags that represent unique subset of -// endpoints type ZoneIngress_AvailableService struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -197,7 +206,7 @@ var file_mesh_v1alpha1_zone_ingress_proto_rawDesc = []byte{ 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, - 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc1, 0x03, 0x0a, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd5, 0x03, 0x0a, 0x0b, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, @@ -207,29 +216,30 @@ var file_mesh_v1alpha1_zone_ingress_proto_rawDesc = []byte{ 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x5e, 0x0a, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, - 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, - 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x11, 0x61, - 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x1a, 0xcd, 0x01, 0x0a, 0x10, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, - 0x72, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, - 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, - 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x5e, 0x0a, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x52, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x1a, 0xcd, 0x01, 0x0a, 0x10, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, + 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x61, 0x67, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, + 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, + 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x73, + 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x69, 0x6e, + 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x1a, 0x37, 0x0a, 0x09, 0x54, + 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, + 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/mesh/v1alpha1/zone_ingress.proto b/api/mesh/v1alpha1/zone_ingress.proto index fdb7de95ec74..b042ac59671f 100644 --- a/api/mesh/v1alpha1/zone_ingress.proto +++ b/api/mesh/v1alpha1/zone_ingress.proto @@ -27,8 +27,10 @@ message ZoneIngress { // Kuma clusters. uint32 advertisedPort = 4; - // AvailableService contains tags that represent unique subset of - // endpoints + // Zone field contains Zone name where ingress is serving, field will be + // automatically set by Global Kuma CP + string zone = 6; + message AvailableService { // tags of the service map tags = 1; @@ -37,5 +39,8 @@ message ZoneIngress { // mesh of the instances available for given tags string mesh = 3; } + + // AvailableService contains tags that represent unique subset of + // endpoints repeated AvailableService availableServices = 5; } diff --git a/app/kuma-dp/cmd/dataplane.go b/app/kuma-dp/cmd/dataplane.go index d9df3a93d6d7..25e260831e90 100644 --- a/app/kuma-dp/cmd/dataplane.go +++ b/app/kuma-dp/cmd/dataplane.go @@ -8,12 +8,20 @@ import ( kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" - "github.com/kumahq/kuma/pkg/core/resources/model" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/resources/model/rest" "github.com/kumahq/kuma/pkg/util/template" ) func readDataplaneResource(cmd *cobra.Command, cfg *kuma_dp.Config) (*rest.Resource, error) { + return readResource(cmd, cfg, core_mesh.DataplaneType, core_model.ScopeMesh) +} + +func readZoneIngressResource(cmd *cobra.Command, cfg *kuma_dp.Config) (*rest.Resource, error) { + return readResource(cmd, cfg, core_mesh.ZoneIngressType, core_model.ScopeGlobal) +} + +func readResource(cmd *cobra.Command, cfg *kuma_dp.Config, typ core_model.ResourceType, scope core_model.ResourceScope) (*rest.Resource, error) { var b []byte var err error // load from file first @@ -36,16 +44,16 @@ func readDataplaneResource(cmd *cobra.Command, cfg *kuma_dp.Config) (*rest.Resou } b = template.Render(string(b), cfg.DataplaneRuntime.ResourceVars) - runLog.Info("rendered dataplane", "dataplane", string(b)) + runLog.Info("rendered resource", "resource", string(b)) res, err := rest.Unmarshall(b) if err != nil { return nil, err } - if res.Meta.Type != string(core_mesh.DataplaneType) { - return nil, errors.Errorf("invalid resource of type: %s. Expected: Dataplane", res.Meta.Type) + if res.Meta.Type != string(typ) { + return nil, errors.Errorf("invalid resource of type: %s. Expected: %s", res.Meta.Type, typ) } - if err := core_mesh.ValidateMeta(res.Meta.GetName(), res.Meta.GetMesh(), model.ScopeMesh); err.HasViolations() { + if err := core_mesh.ValidateMeta(res.Meta.GetName(), res.Meta.GetMesh(), scope); err.HasViolations() { return nil, &err } diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index 4385a8d552aa..804ec483ec5e 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -5,15 +5,15 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" + "github.com/spf13/cobra" + kumadp_config "github.com/kumahq/kuma/app/kuma-dp/pkg/config" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics" "github.com/kumahq/kuma/pkg/core/resources/model/rest" "github.com/kumahq/kuma/pkg/core/runtime/component" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/accesslogs" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy" "github.com/kumahq/kuma/pkg/config" @@ -31,7 +31,7 @@ var runLog = dataplaneLog.WithName("run") // To extend Kuma DP, plug your code in RunE. Use RootContext.Config and add components to RootContext.ComponentManager func newRunCmd(rootCtx *RootContext) *cobra.Command { cfg := rootCtx.Config - var dp *rest.Resource + var proxyResource *rest.Resource var tmpDir string var adminPort uint32 cmd := &cobra.Command{ @@ -55,17 +55,26 @@ func newRunCmd(rootCtx *RootContext) *cobra.Command { } var err error - dp, err = readDataplaneResource(cmd, cfg) - if err != nil { - runLog.Error(err, "unable to read provided dataplane") - return err + if cfg.Dataplane.IsIngress { + proxyResource, err = readZoneIngressResource(cmd, cfg) + if err != nil { + runLog.Error(err, "unable to read provided zone ingress") + return err + } + } else { + proxyResource, err = readDataplaneResource(cmd, cfg) + if err != nil { + runLog.Error(err, "unable to read provided dataplane") + return err + } } - if dp != nil { + + if proxyResource != nil { if cfg.Dataplane.Name != "" || cfg.Dataplane.Mesh != "" { return errors.New("--name and --mesh cannot be specified when dataplane definition is provided. Mesh and name will be read from the dataplane definition.") } - cfg.Dataplane.Mesh = dp.Meta.GetMesh() - cfg.Dataplane.Name = dp.Meta.GetName() + cfg.Dataplane.Mesh = proxyResource.Meta.GetMesh() + cfg.Dataplane.Name = proxyResource.Meta.GetName() } if !cfg.Dataplane.AdminPort.Empty() { @@ -137,7 +146,7 @@ func newRunCmd(rootCtx *RootContext) *cobra.Command { opts := envoy.Opts{ Config: *cfg, Generator: rootCtx.BootstrapGenerator, - Dataplane: dp, + Dataplane: proxyResource, DynamicMetadata: rootCtx.BootstrapDynamicMetadata, Stdout: cmd.OutOrStdout(), Stderr: cmd.OutOrStderr(), @@ -191,6 +200,7 @@ func newRunCmd(rootCtx *RootContext) *cobra.Command { cmd.PersistentFlags().StringVar(&cfg.Dataplane.Name, "name", cfg.Dataplane.Name, "Name of the Dataplane") cmd.PersistentFlags().Var(&cfg.Dataplane.AdminPort, "admin-port", `Port (or range of ports to choose from) for Envoy Admin API to listen on. Empty value indicates that Envoy Admin API should not be exposed over TCP. Format: "9901 | 9901-9999 | 9901- | -9901"`) cmd.PersistentFlags().StringVar(&cfg.Dataplane.Mesh, "mesh", cfg.Dataplane.Mesh, "Mesh that Dataplane belongs to") + cmd.PersistentFlags().BoolVar(&cfg.Dataplane.IsIngress, "ingress", false, "If true then kuma-dp will register itself as a ZoneIngress") cmd.PersistentFlags().StringVar(&cfg.ControlPlane.URL, "cp-address", cfg.ControlPlane.URL, "URL of the Control Plane Dataplane Server. Example: https://localhost:5678") cmd.PersistentFlags().StringVar(&cfg.ControlPlane.CaCertFile, "ca-cert-file", cfg.ControlPlane.CaCertFile, "Path to CA cert by which connection to the Control Plane will be verified if HTTPS is used") cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.BinaryPath, "binary-path", cfg.DataplaneRuntime.BinaryPath, "Binary path of Envoy executable") diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index 6a0e30a30758..0ccd07a44fe9 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -14,6 +14,8 @@ import ( "github.com/pkg/errors" "github.com/sethvargo/go-retry" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp" "github.com/kumahq/kuma/pkg/core" kuma_version "github.com/kumahq/kuma/pkg/version" @@ -116,9 +118,14 @@ func (b *remoteBootstrap) requestForBootstrap(url *net_url.URL, cfg kuma_dp.Conf if cfg.DataplaneRuntime.Token != "" { token = cfg.DataplaneRuntime.Token } + proxyType := mesh_proto.RegularDpType + if cfg.Dataplane.IsIngress { + proxyType = mesh_proto.IngressDpType + } request := types.BootstrapRequest{ - Mesh: cfg.Dataplane.Mesh, - Name: cfg.Dataplane.Name, + Mesh: cfg.Dataplane.Mesh, + Name: cfg.Dataplane.Name, + ProxyType: string(proxyType), // if not set in config, the 0 will be sent which will result in providing default admin port // that is set in the control plane bootstrap params AdminPort: cfg.Dataplane.AdminPort.Lowest(), diff --git a/app/kumactl/pkg/tokens/client_test.go b/app/kumactl/pkg/tokens/client_test.go index 571b00e4ac04..7002dc711954 100644 --- a/app/kumactl/pkg/tokens/client_test.go +++ b/app/kumactl/pkg/tokens/client_test.go @@ -10,6 +10,8 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" + "github.com/kumahq/kuma/app/kumactl/pkg/tokens" config_kumactl "github.com/kumahq/kuma/pkg/config/app/kumactl/v1alpha1" "github.com/kumahq/kuma/pkg/tokens/builtin/issuer" @@ -29,13 +31,26 @@ func (s *staticTokenIssuer) Validate(token issuer.Token, meshName string) (issue return issuer.DataplaneIdentity{}, errors.New("not implemented") } +type zoneIngressStaticTokenIssuer struct { +} + +var _ zoneingress.TokenIssuer = &zoneIngressStaticTokenIssuer{} + +func (z *zoneIngressStaticTokenIssuer) Generate(identity zoneingress.Identity) (zoneingress.Token, error) { + return fmt.Sprintf("token-for-%s", identity.Name), nil +} + +func (z *zoneIngressStaticTokenIssuer) Validate(token zoneingress.Token) (zoneingress.Identity, error) { + return zoneingress.Identity{}, errors.New("not implemented") +} + var _ = Describe("Tokens Client", func() { var server *httptest.Server BeforeEach(func() { container := restful.NewContainer() - container.Add(tokens_server.NewWebservice(&staticTokenIssuer{})) + container.Add(tokens_server.NewWebservice(&staticTokenIssuer{}, &zoneIngressStaticTokenIssuer{})) server = httptest.NewServer(container.ServeMux) }) diff --git a/deployments/charts/kuma/crds/zoneingresses.yaml b/deployments/charts/kuma/crds/zoneingresses.yaml new file mode 100644 index 000000000000..4ac80539c070 --- /dev/null +++ b/deployments/charts/kuma/crds/zoneingresses.yaml @@ -0,0 +1,27 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object diff --git a/deployments/charts/kuma/templates/cp-rbac.yaml b/deployments/charts/kuma/templates/cp-rbac.yaml index 20dcd017e489..e8360acfd127 100644 --- a/deployments/charts/kuma/templates/cp-rbac.yaml +++ b/deployments/charts/kuma/templates/cp-rbac.yaml @@ -50,6 +50,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates diff --git a/deployments/charts/kuma/templates/ingress-deployment.yaml b/deployments/charts/kuma/templates/ingress-deployment.yaml index ef005375d36e..4cb848f25455 100644 --- a/deployments/charts/kuma/templates/ingress-deployment.yaml +++ b/deployments/charts/kuma/templates/ingress-deployment.yaml @@ -53,8 +53,6 @@ spec: value: "https://{{ include "kuma.controlPlane.serviceName" . }}.{{ .Release.Namespace }}:5678" - name: KUMA_CONTROL_PLANE_CA_CERT_FILE value: /var/run/secrets/kuma.io/tls-cert/ca.crt - - name: KUMA_DATAPLANE_MESH - value: {{ .Values.ingress.mesh }} - name: KUMA_DATAPLANE_NAME value: $(POD_NAME).$(POD_NAMESPACE) - name: KUMA_DATAPLANE_ADMIN_PORT @@ -63,6 +61,8 @@ spec: value: {{ .Values.ingress.drainTime }} - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: KUMA_DATAPLANE_IS_INGRESS + value: "true" args: - run - --log-level=info diff --git a/pkg/api-server/definitions/all.go b/pkg/api-server/definitions/all.go index e024df6217b3..92127cf3a24b 100644 --- a/pkg/api-server/definitions/all.go +++ b/pkg/api-server/definitions/all.go @@ -21,6 +21,7 @@ var DefaultCRUDLEndpoints = []ResourceWsDefinition{ CircuitBreakerWsDefinition, ZoneWsDefinition, ZoneInsightWsDefinition, + ZoneIngressWsDefinition, SecretWsDefinition, GlobalSecretWsDefinition, RetryWsDefinition, diff --git a/pkg/api-server/definitions/zone-ingress.go b/pkg/api-server/definitions/zone-ingress.go new file mode 100644 index 000000000000..9e353d672040 --- /dev/null +++ b/pkg/api-server/definitions/zone-ingress.go @@ -0,0 +1,17 @@ +package definitions + +import ( + "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + "github.com/kumahq/kuma/pkg/core/resources/model" +) + +var ZoneIngressWsDefinition = ResourceWsDefinition{ + Name: "ZoneIngress", + Path: "zone-ingresses", + ResourceFactory: func() model.Resource { + return mesh.NewZoneIngressResource() + }, + ResourceListFactory: func() model.ResourceList { + return &mesh.ZoneIngressResourceList{} + }, +} diff --git a/pkg/api-server/server.go b/pkg/api-server/server.go index 155704da17de..ce37c399fc41 100644 --- a/pkg/api-server/server.go +++ b/pkg/api-server/server.go @@ -205,12 +205,16 @@ func addResourcesEndpoints(ws *restful.WebService, defs []definitions.ResourceWs } func dataplaneTokenWs(resManager manager.ResourceManager, cfg *kuma_cp.Config) (*restful.WebService, error) { - generator, err := builtin.NewDataplaneTokenIssuer(resManager) + dpIssuer, err := builtin.NewDataplaneTokenIssuer(resManager) + if err != nil { + return nil, err + } + zoneIngressIssuer, err := builtin.NewZoneIngressTokenIssuer(resManager) if err != nil { return nil, err } adminAuth := authz.AdminAuth{AllowFromLocalhost: cfg.ApiServer.Auth.AllowFromLocalhost} - return tokens_server.NewWebservice(generator).Filter(adminAuth.Validate), nil + return tokens_server.NewWebservice(dpIssuer, zoneIngressIssuer).Filter(adminAuth.Validate), nil } func (a *ApiServer) Start(stop <-chan struct{}) error { diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index efb0ee178ba8..a69c36ef2fd0 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -109,6 +109,8 @@ type Dataplane struct { Mesh string `yaml:"mesh,omitempty" envconfig:"kuma_dataplane_mesh"` // Dataplane name. Name string `yaml:"name,omitempty" envconfig:"kuma_dataplane_name"` + // IsIngress if true, runs kuma-dp as a zone ingress + IsIngress bool `yaml:"isIngress,omitempty" envconfig:"kuma_dataplane_is_ingress"` // Port (or range of ports to choose from) for Envoy Admin API to listen on. // Empty value indicates that Envoy Admin API should not be exposed over TCP. // Format: "9901 | 9901-9999 | 9901- | -9901". @@ -182,7 +184,7 @@ func (d *Dataplane) Sanitize() { } func (d *Dataplane) Validate() (errs error) { - if d.Mesh == "" { + if d.Mesh == "" && !d.IsIngress { errs = multierr.Append(errs, errors.Errorf(".Mesh must be non-empty")) } if d.Name == "" { diff --git a/pkg/core/bootstrap/bootstrap.go b/pkg/core/bootstrap/bootstrap.go index 6c04f7e5ec7a..da5530d6611c 100644 --- a/pkg/core/bootstrap/bootstrap.go +++ b/pkg/core/bootstrap/bootstrap.go @@ -70,8 +70,9 @@ func buildRuntime(cfg kuma_cp.Config, closeCh <-chan struct{}) (core_runtime.Run } // we add Secret store to unified ResourceStore so global<->remote synchronizer can use unified interface builder.WithResourceStore(core_store.NewCustomizableResourceStore(builder.ResourceStore(), map[core_model.ResourceType]core_store.ResourceStore{ - system.SecretType: builder.SecretStore(), - system.ConfigType: builder.ConfigStore(), + system.SecretType: builder.SecretStore(), + system.GlobalSecretType: builder.SecretStore(), + system.ConfigType: builder.ConfigStore(), })) if err := initializeConfigManager(cfg, builder); err != nil { diff --git a/pkg/core/resources/apis/mesh/zone_ingress.go b/pkg/core/resources/apis/mesh/zone_ingress.go index 8be2c59725d4..3495a9e9c046 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress.go +++ b/pkg/core/resources/apis/mesh/zone_ingress.go @@ -25,29 +25,29 @@ func NewZoneIngressResource() *ZoneIngressResource { } } -func (t *ZoneIngressResource) GetType() model.ResourceType { +func (r *ZoneIngressResource) GetType() model.ResourceType { return ZoneIngressType } -func (t *ZoneIngressResource) GetMeta() model.ResourceMeta { - return t.Meta +func (r *ZoneIngressResource) GetMeta() model.ResourceMeta { + return r.Meta } -func (t *ZoneIngressResource) SetMeta(m model.ResourceMeta) { - t.Meta = m +func (r *ZoneIngressResource) SetMeta(m model.ResourceMeta) { + r.Meta = m } -func (t *ZoneIngressResource) GetSpec() model.ResourceSpec { - return t.Spec +func (r *ZoneIngressResource) GetSpec() model.ResourceSpec { + return r.Spec } -func (t *ZoneIngressResource) SetSpec(spec model.ResourceSpec) error { +func (r *ZoneIngressResource) SetSpec(spec model.ResourceSpec) error { zoneIngress, ok := spec.(*mesh_proto.ZoneIngress) if !ok { return errors.New("invalid type of spec") } else { - t.Spec = zoneIngress + r.Spec = zoneIngress return nil } } -func (t *ZoneIngressResource) Scope() model.ResourceScope { +func (r *ZoneIngressResource) Scope() model.ResourceScope { return model.ScopeGlobal } diff --git a/pkg/core/resources/apis/mesh/zone_ingress_helpers.go b/pkg/core/resources/apis/mesh/zone_ingress_helpers.go new file mode 100644 index 000000000000..9dba4958849f --- /dev/null +++ b/pkg/core/resources/apis/mesh/zone_ingress_helpers.go @@ -0,0 +1,69 @@ +package mesh + +import ( + "net" + + "github.com/pkg/errors" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" +) + +func (r *ZoneIngressResource) UsesInboundInterface(address net.IP, port uint32) bool { + if r == nil { + return false + } + if port == r.Spec.Port && overlap(address, net.ParseIP(r.Spec.Address)) { + return true + } + if port == r.Spec.AdvertisedPort && overlap(address, net.ParseIP(r.Spec.AdvertisedAddress)) { + return true + } + return false +} + +func (r *ZoneIngressResource) IsRemoteIngress(localZone string) bool { + if r.Spec.GetZone() == "" || r.Spec.GetZone() == localZone { + return false + } + return true +} + +func (r *ZoneIngressResource) HasPublicAddress() bool { + if r == nil { + return false + } + return r.Spec.GetAdvertisedAddress() != "" && r.Spec.GetAdvertisedPort() != 0 +} + +func NewZoneIngressResourceFromDataplane(dataplane *DataplaneResource) (*ZoneIngressResource, error) { + spec, err := convert(dataplane.Spec) + if err != nil { + return nil, err + } + return &ZoneIngressResource{ + Meta: dataplane.Meta, + Spec: spec, + }, nil +} + +func convert(dataplane *mesh_proto.Dataplane) (*mesh_proto.ZoneIngress, error) { + if !dataplane.IsIngress() { + return nil, errors.New("provided dataplane is not an ingress") + } + if len(dataplane.GetNetworking().Inbound) == 0 { + return nil, errors.New("provided dataplane is not an ingress") + } + var availableServices []*mesh_proto.ZoneIngress_AvailableService + for _, as := range dataplane.GetNetworking().GetIngress().GetAvailableServices() { + availableServices = append(availableServices, &mesh_proto.ZoneIngress_AvailableService{ + Tags: as.GetTags(), + Instances: as.GetInstances(), + Mesh: as.GetMesh(), + }) + } + return &mesh_proto.ZoneIngress{ + Address: dataplane.GetNetworking().GetAddress(), + Port: dataplane.GetNetworking().Inbound[0].GetPort(), + AvailableServices: availableServices, + }, nil +} diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator.go b/pkg/core/resources/apis/mesh/zone_ingress_validator.go index 5b95d2a35190..f1bcbee35bea 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_validator.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator.go @@ -5,9 +5,9 @@ import ( "github.com/kumahq/kuma/pkg/core/validators" ) -func (d *ZoneIngressResource) Validate() error { +func (r *ZoneIngressResource) Validate() error { var err validators.ValidationError - err.Add(validateZoneIngress(validators.RootedAt("ingress"), d.Spec)) + err.Add(validateZoneIngress(validators.RootedAt("ingress"), r.Spec)) return err.OrNil() } diff --git a/pkg/core/xds/metadata.go b/pkg/core/xds/metadata.go index 12bf74bf87e6..8068779e0495 100644 --- a/pkg/core/xds/metadata.go +++ b/pkg/core/xds/metadata.go @@ -40,13 +40,14 @@ const ( // This way, xDS server will be able to use Envoy node metadata // to generate xDS resources that depend on environment-specific configuration. type DataplaneMetadata struct { - DataplaneTokenPath string - DataplaneToken string - DataplaneResource *core_mesh.DataplaneResource - AdminPort uint32 - DNSPort uint32 - EmptyDNSPort uint32 - DynamicMetadata map[string]string + DataplaneTokenPath string + DataplaneToken string + DataplaneResource *core_mesh.DataplaneResource + ZoneIngressResource *core_mesh.ZoneIngressResource + AdminPort uint32 + DNSPort uint32 + EmptyDNSPort uint32 + DynamicMetadata map[string]string } func (m *DataplaneMetadata) GetDataplaneTokenPath() string { @@ -70,6 +71,13 @@ func (m *DataplaneMetadata) GetDataplaneResource() *core_mesh.DataplaneResource return m.DataplaneResource } +func (m *DataplaneMetadata) GetZoneIngressResource() *core_mesh.ZoneIngressResource { + if m == nil { + return nil + } + return m.ZoneIngressResource +} + func (m *DataplaneMetadata) GetAdminPort() uint32 { if m == nil { return 0 @@ -117,11 +125,14 @@ func DataplaneMetadataFromXdsMetadata(xdsMetadata *_struct.Struct) *DataplaneMet if err != nil { metadataLog.Error(err, "invalid value in dataplane metadata", "field", fieldDataplaneDataplaneResource, "value", value) } - dp, ok := res.(*core_mesh.DataplaneResource) - if !ok { + switch r := res.(type) { + case *core_mesh.DataplaneResource: + metadata.DataplaneResource = r + case *core_mesh.ZoneIngressResource: + metadata.ZoneIngressResource = r + default: metadataLog.Error(err, "invalid value in dataplane metadata", "field", fieldDataplaneDataplaneResource, "value", value) } - metadata.DataplaneResource = dp } if value := xdsMetadata.Fields[fieldDynamicMetadata]; value != nil { dynamicMetadata := map[string]string{} diff --git a/pkg/core/xds/types.go b/pkg/core/xds/types.go index df30404349d0..ded154a3f97a 100644 --- a/pkg/core/xds/types.go +++ b/pkg/core/xds/types.go @@ -20,12 +20,13 @@ import ( type StreamID = int64 type ProxyId struct { - Mesh string - Name string + Mesh string + Name string + ProxyType mesh_proto.DpType } func (id ProxyId) String() string { - return fmt.Sprintf("%s.%s", id.Mesh, id.Name) + return fmt.Sprintf("%s.%s:%s", id.Mesh, id.Name, id.ProxyType) } // ServiceName is a convenience type alias to clarify the meaning of string value. @@ -203,12 +204,44 @@ func (l EndpointList) Filter(selector mesh_proto.TagSelector) EndpointList { return endpoints } -func BuildProxyId(mesh, name string, more ...string) (*ProxyId, error) { - id := strings.Join(append([]string{mesh, name}, more...), ".") - return ParseProxyIdFromString(id) +func BuildProxyId(mesh, name string, proxyType mesh_proto.DpType) *ProxyId { + return &ProxyId{ + Name: name, + Mesh: mesh, + ProxyType: proxyType, + } } func ParseProxyIdFromString(id string) (*ProxyId, error) { + if id == "" { + return nil, errors.Errorf("Envoy ID must not be nil") + } + parts := strings.SplitN(id, ":", 2) + if len(parts) == 1 { + proxyId, err := ParseProxyIdFromStringOldFormat(id) + if err != nil { + return nil, err + } + proxyId.ProxyType = mesh_proto.RegularDpType + return proxyId, nil + } + proxyType := mesh_proto.DpType(parts[1]) + meshAndName := strings.SplitN(parts[0], ".", 2) + mesh, name := meshAndName[0], meshAndName[1] + if name == "" { + return nil, errors.New("name must not be empty") + } + if mesh == "" && proxyType != mesh_proto.IngressDpType { + return nil, errors.New("mesh must not be empty") + } + return &ProxyId{ + Mesh: mesh, + Name: name, + ProxyType: proxyType, + }, nil +} + +func ParseProxyIdFromStringOldFormat(id string) (*ProxyId, error) { if id == "" { return nil, errors.Errorf("Envoy ID must not be nil") } @@ -237,9 +270,10 @@ func (id *ProxyId) ToResourceKey() core_model.ResourceKey { } } -func FromResourceKey(key core_model.ResourceKey) ProxyId { +func FromResourceKey(proxyType mesh_proto.DpType, key core_model.ResourceKey) ProxyId { return ProxyId{ - Mesh: key.Mesh, - Name: key.Name, + Mesh: key.Mesh, + Name: key.Name, + ProxyType: proxyType, } } diff --git a/pkg/core/xds/types_test.go b/pkg/core/xds/types_test.go index 929501679ce2..704292690a11 100644 --- a/pkg/core/xds/types_test.go +++ b/pkg/core/xds/types_test.go @@ -32,13 +32,25 @@ var _ = Describe("xDS", func() { Entry("mesh and name without namespace", testCase{ nodeID: "demo.example", expected: core_xds.ProxyId{ - Mesh: "demo", Name: "example", + Mesh: "demo", Name: "example", ProxyType: mesh_proto.RegularDpType, }, }), Entry("name with namespace and mesh", testCase{ nodeID: "demo.example.sample", expected: core_xds.ProxyId{ - Mesh: "demo", Name: "example.sample", + Mesh: "demo", Name: "example.sample", ProxyType: mesh_proto.RegularDpType, + }, + }), + Entry("mesh and name without namespace and proxy type", testCase{ + nodeID: "demo.example:ingress", + expected: core_xds.ProxyId{ + Mesh: "demo", Name: "example", ProxyType: mesh_proto.IngressDpType, + }, + }), + Entry("name with namespace and mesh and proxy type", testCase{ + nodeID: "demo.example.sample:ingress", + expected: core_xds.ProxyId{ + Mesh: "demo", Name: "example.sample", ProxyType: mesh_proto.IngressDpType, }, }), ) diff --git a/pkg/defaults/components.go b/pkg/defaults/components.go index 839a2b12d72c..20aa9b98e955 100644 --- a/pkg/defaults/components.go +++ b/pkg/defaults/components.go @@ -2,10 +2,12 @@ package defaults import ( "context" + "sync" "time" "github.com/pkg/errors" "github.com/sethvargo/go-retry" + "go.uber.org/multierr" kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp" config_core "github.com/kumahq/kuma/pkg/config/core" @@ -48,22 +50,61 @@ func (d *defaultsComponent) NeedLeaderElection() bool { return true } -func (d *defaultsComponent) Start(_ <-chan struct{}) error { +func (d *defaultsComponent) Start(stop <-chan struct{}) error { // todo(jakubdyszkiewicz) once this https://github.com/kumahq/kuma/issues/1001 is done. Wait for all the components to be ready. + ctx, cancelFn := context.WithCancel(context.Background()) + defer cancelFn() + wg := &sync.WaitGroup{} + errChan := make(chan error) + if d.config.SkipMeshCreation { log.V(1).Info("skipping default Mesh creation because KUMA_DEFAULTS_SKIP_MESH_CREATION is set to true") - } else if err := doWithRetry(d.createMeshIfNotExist); err != nil { - // Retry this operation since on Kubernetes Mesh needs to be validated and set default values. - // This code can execute before the control plane is ready therefore hooks can fail. - return errors.Wrap(err, "could not create the default Mesh") + } else { + wg.Add(1) + go func() { + defer wg.Done() + if err := doWithRetry(ctx, d.createMeshIfNotExist); err != nil { + // Retry this operation since on Kubernetes Mesh needs to be validated and set default values. + // This code can execute before the control plane is ready therefore hooks can fail. + errChan <- errors.Wrap(err, "could not create the default Mesh") + } + }() + } + + wg.Add(1) + go func() { + defer wg.Done() + if err := doWithRetry(ctx, d.createZoneIngressSigningKeyIfNotExist); err != nil { + // Retry this operation since on Kubernetes Mesh needs to be validated and set default values. + // This code can execute before the control plane is ready therefore hooks can fail. + errChan <- errors.Wrap(err, "could not create the default Control Plane Token's Signing Key") + } + }() + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + close(errChan) + }() + + var errs error + for { + select { + case <-stop: + return errs + case err := <-errChan: + errs = multierr.Append(errs, err) + case <-done: + return errs + } } - return nil } -func doWithRetry(fn func() error) error { +func doWithRetry(ctx context.Context, fn func(context.Context) error) error { backoff, _ := retry.NewConstant(5 * time.Second) backoff = retry.WithMaxDuration(10*time.Minute, backoff) // if after this time we cannot create a resource - something is wrong and we should return an error which will restart CP. - return retry.Do(context.Background(), backoff, func(ctx context.Context) error { - return retry.RetryableError(fn()) // retry all errors + return retry.Do(ctx, backoff, func(ctx context.Context) error { + return retry.RetryableError(fn(ctx)) // retry all errors }) } diff --git a/pkg/defaults/components_test.go b/pkg/defaults/components_test.go index 1ed163385979..07c5b654fd82 100644 --- a/pkg/defaults/components_test.go +++ b/pkg/defaults/components_test.go @@ -5,13 +5,19 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "google.golang.org/protobuf/types/known/wrapperspb" + + system_proto "github.com/kumahq/kuma/api/system/v1alpha1" + "github.com/kumahq/kuma/pkg/core/resources/apis/system" + "github.com/kumahq/kuma/pkg/core/secrets/cipher" + secret_manager "github.com/kumahq/kuma/pkg/core/secrets/manager" "github.com/kumahq/kuma/api/mesh/v1alpha1" kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp" "github.com/kumahq/kuma/pkg/config/core" mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_manager "github.com/kumahq/kuma/pkg/core/resources/manager" - "github.com/kumahq/kuma/pkg/core/resources/model" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_store "github.com/kumahq/kuma/pkg/core/resources/store" core_component "github.com/kumahq/kuma/pkg/core/runtime/component" "github.com/kumahq/kuma/pkg/defaults" @@ -40,7 +46,7 @@ var _ = Describe("Defaults Component", func() { // then Expect(err).ToNot(HaveOccurred()) - err = manager.Get(context.Background(), mesh_core.NewMeshResource(), core_store.GetByKey(model.DefaultMesh, model.NoMesh)) + err = manager.Get(context.Background(), mesh_core.NewMeshResource(), core_store.GetByKey(core_model.DefaultMesh, core_model.NoMesh)) Expect(err).ToNot(HaveOccurred()) }) @@ -59,7 +65,7 @@ var _ = Describe("Defaults Component", func() { }, }, } - err := manager.Create(context.Background(), mesh, core_store.CreateByKey(model.DefaultMesh, model.NoMesh)) + err := manager.Create(context.Background(), mesh, core_store.CreateByKey(core_model.DefaultMesh, core_model.NoMesh)) Expect(err).ToNot(HaveOccurred()) // when @@ -68,7 +74,7 @@ var _ = Describe("Defaults Component", func() { // then mesh = mesh_core.NewMeshResource() Expect(err).ToNot(HaveOccurred()) - err = manager.Get(context.Background(), mesh, core_store.GetByKey(model.DefaultMesh, model.NoMesh)) + err = manager.Get(context.Background(), mesh, core_store.GetByKey(core_model.DefaultMesh, core_model.NoMesh)) Expect(err).ToNot(HaveOccurred()) Expect(mesh.Spec.Mtls.EnabledBackend).To(Equal("builtin")) }) @@ -99,4 +105,53 @@ var _ = Describe("Defaults Component", func() { }) }) + Describe("zone ingress signing key creation", func() { + + var component core_component.Component + var manager core_manager.ResourceManager + + BeforeEach(func() { + cfg := &kuma_cp.Defaults{} + store := resources_memory.NewStore() + defaultManager := core_manager.NewResourceManager(store) + customManagers := map[core_model.ResourceType]core_manager.ResourceManager{} + customManagers[system.GlobalSecretType] = secret_manager.NewGlobalSecretManager(store, cipher.None()) + manager = core_manager.NewCustomizableResourceManager(defaultManager, customManagers) + component = defaults.NewDefaultsComponent(cfg, core.Standalone, core.UniversalEnvironment, manager, store) + }) + + It("should create zone ingress signing key and default mesh", func() { + // when + err := component.Start(nil) + + // then + Expect(err).ToNot(HaveOccurred()) + err = manager.Get(context.Background(), system.NewGlobalSecretResource(), core_store.GetByKey("zone-ingress-token-signing-key", core_model.NoMesh)) + Expect(err).ToNot(HaveOccurred()) + err = manager.Get(context.Background(), mesh_core.NewMeshResource(), core_store.GetByKey(core_model.DefaultMesh, core_model.NoMesh)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not override already created signing key", func() { + // given + signingKey := &system.GlobalSecretResource{ + Spec: &system_proto.Secret{ + Data: wrapperspb.Bytes([]byte("hello")), + }, + } + err := manager.Create(context.Background(), signingKey, core_store.CreateByKey("zone-ingress-token-signing-key", core_model.NoMesh)) + Expect(err).ToNot(HaveOccurred()) + + // when + err = component.Start(nil) + + // then + Expect(err).ToNot(HaveOccurred()) + actual := system.NewGlobalSecretResource() + err = manager.Get(context.Background(), actual, core_store.GetByKey("zone-ingress-token-signing-key", core_model.NoMesh)) + Expect(err).ToNot(HaveOccurred()) + Expect(actual.Spec.GetData().GetValue()).To(Equal([]byte("hello"))) + }) + }) + }) diff --git a/pkg/defaults/mesh.go b/pkg/defaults/mesh.go index 35cac3ec813d..484d4d27f5a2 100644 --- a/pkg/defaults/mesh.go +++ b/pkg/defaults/mesh.go @@ -12,9 +12,9 @@ var defaultMeshKey = core_model.ResourceKey{ Name: core_model.DefaultMesh, } -func (d *defaultsComponent) createMeshIfNotExist() error { +func (d *defaultsComponent) createMeshIfNotExist(ctx context.Context) error { mesh := mesh_core.NewMeshResource() - err := d.resManager.Get(context.Background(), mesh, core_store.GetBy(defaultMeshKey)) + err := d.resManager.Get(ctx, mesh, core_store.GetBy(defaultMeshKey)) if err == nil { log.V(1).Info("default Mesh already exists. Skip creating default Mesh.") return nil @@ -23,7 +23,7 @@ func (d *defaultsComponent) createMeshIfNotExist() error { return err } log.Info("trying to create default Mesh") - if err := d.resManager.Create(context.Background(), mesh, core_store.CreateBy(defaultMeshKey)); err != nil { + if err := d.resManager.Create(ctx, mesh, core_store.CreateBy(defaultMeshKey)); err != nil { log.V(1).Info("could not create default mesh", "err", err) return err } diff --git a/pkg/defaults/signing_key.go b/pkg/defaults/signing_key.go new file mode 100644 index 000000000000..28558e331ada --- /dev/null +++ b/pkg/defaults/signing_key.go @@ -0,0 +1,29 @@ +package defaults + +import ( + "context" + + "github.com/pkg/errors" + + core_store "github.com/kumahq/kuma/pkg/core/resources/store" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" +) + +func (d *defaultsComponent) createZoneIngressSigningKeyIfNotExist(ctx context.Context) error { + signingKey, err := zoneingress.CreateSigningKey() + if err != nil { + return errors.Wrap(err, "could not create a signing key") + } + key := zoneingress.SigningKeyResourceKey() + err = d.resManager.Get(ctx, signingKey, core_store.GetBy(key)) + if err == nil { + return nil + } + if !core_store.IsResourceNotFound(err) { + return errors.Wrap(err, "could not retrieve a resource") + } + if err := d.resManager.Create(ctx, signingKey, core_store.CreateBy(key)); err != nil { + return errors.Wrap(err, "could not create a resource") + } + return nil +} diff --git a/pkg/dns/outbound.go b/pkg/dns/outbound.go index e2462aef7b84..07ce23f75a6c 100644 --- a/pkg/dns/outbound.go +++ b/pkg/dns/outbound.go @@ -19,6 +19,7 @@ const VIPListenPort = uint32(80) func VIPOutbounds( resourceKey model.ResourceKey, dataplanes []*core_mesh.DataplaneResource, + zoneIngresses []*core_mesh.ZoneIngressResource, vips vips.List, externalServices []*core_mesh.ExternalServiceResource, ) []*mesh_proto.Dataplane_Networking_Outbound { @@ -29,8 +30,9 @@ func VIPOutbounds( serviceVIPMap := map[string]vipEntry{} services := []string{} for _, dataplane := range dataplanes { + // backwards compatibility if dataplane.Spec.IsIngress() { - for _, service := range dataplane.Spec.Networking.Ingress.AvailableServices { + for _, service := range dataplane.Spec.GetNetworking().GetIngress().GetAvailableServices() { if service.Mesh == resourceKey.Mesh { // Only add outbounds for services in the same mesh inService := service.Tags[mesh_proto.ServiceTag] @@ -44,7 +46,7 @@ func VIPOutbounds( } } } else { - for _, inbound := range dataplane.Spec.Networking.Inbound { + for _, inbound := range dataplane.Spec.GetNetworking().GetInbound() { inService := inbound.GetTags()[mesh_proto.ServiceTag] if _, found := serviceVIPMap[inService]; !found { vip, err := ForwardLookup(vips, inService) @@ -57,6 +59,22 @@ func VIPOutbounds( } } + for _, zi := range zoneIngresses { + for _, service := range zi.Spec.GetAvailableServices() { + if service.Mesh == resourceKey.Mesh { + // Only add outbounds for services in the same mesh + inService := service.Tags[mesh_proto.ServiceTag] + if _, found := serviceVIPMap[inService]; !found { + vip, err := ForwardLookup(vips, inService) + if err == nil { + serviceVIPMap[inService] = vipEntry{vip, VIPListenPort} + services = append(services, inService) + } + } + } + } + } + for _, externalService := range externalServices { inService := externalService.Spec.Tags[mesh_proto.ServiceTag] if _, found := serviceVIPMap[inService]; !found { diff --git a/pkg/dns/outbound_test.go b/pkg/dns/outbound_test.go index 26e008f06032..3abfc222765d 100644 --- a/pkg/dns/outbound_test.go +++ b/pkg/dns/outbound_test.go @@ -72,7 +72,7 @@ var _ = Describe("VIPOutbounds", func() { } // when - outbounds := dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), dataplanes.Items, vipList, externalServices.Items) + outbounds := dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), dataplanes.Items, nil, vipList, externalServices.Items) // and Expect(outbounds).To(HaveLen(5)) // and @@ -141,7 +141,7 @@ var _ = Describe("VIPOutbounds", func() { } // when - outbounds := dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), dataplanes.Items, vipList, externalServices.Items) + outbounds := dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), dataplanes.Items, nil, vipList, externalServices.Items) // and Expect(outbounds).To(HaveLen(1)) // and @@ -234,7 +234,7 @@ var _ = Describe("VIPOutbounds", func() { vipList["third-external-service"] = "240.0.0.8" actual := &mesh_proto.Dataplane_Networking{} - actual.Outbound = dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), otherDataplanes, vipList, externalServices) + actual.Outbound = dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), otherDataplanes, nil, vipList, externalServices) expected := ` outbound: diff --git a/pkg/dns/vips_allocator.go b/pkg/dns/vips_allocator.go index a0ad434c8cb8..5965270972d1 100644 --- a/pkg/dns/vips_allocator.go +++ b/pkg/dns/vips_allocator.go @@ -172,20 +172,35 @@ func BuildServiceSet(rm manager.ReadOnlyResourceManager, mesh string) (ServiceSe } for _, dp := range filteredDataplanes.Items { + // backwards compatibility if dp.Spec.IsIngress() { - for _, service := range dp.Spec.Networking.Ingress.AvailableServices { + for _, service := range dp.Spec.GetNetworking().GetIngress().GetAvailableServices() { if service.Mesh != mesh { continue } serviceSet[service.Tags[mesh_proto.ServiceTag]] = true } } else { - for _, inbound := range dp.Spec.Networking.Inbound { + for _, inbound := range dp.Spec.GetNetworking().GetInbound() { serviceSet[inbound.GetService()] = true } } } + zoneIngresses := core_mesh.ZoneIngressResourceList{} + if err := rm.List(context.Background(), &zoneIngresses); err != nil { + return nil, err + } + + for _, zi := range zoneIngresses.Items { + for _, service := range zi.Spec.GetAvailableServices() { + if service.Mesh != mesh { + continue + } + serviceSet[service.Tags[mesh_proto.ServiceTag]] = true + } + } + externalServices := core_mesh.ExternalServiceResourceList{} if err := rm.List(context.Background(), &externalServices, store.ListByMesh(mesh)); err != nil { return nil, err diff --git a/pkg/hds/authn/callbacks_test.go b/pkg/hds/authn/callbacks_test.go index b9ede307eec0..b54b3117515b 100644 --- a/pkg/hds/authn/callbacks_test.go +++ b/pkg/hds/authn/callbacks_test.go @@ -38,6 +38,10 @@ func (t *testAuthenticator) Authenticate(ctx context.Context, dataplane *core_me return errors.New("invalid credential") } +func (t *testAuthenticator) AuthenticateZoneIngress(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, credential auth.Credential) error { + return nil +} + var _ = Describe("Authn Callbacks", func() { var testAuth *testAuthenticator diff --git a/pkg/kds/context/context.go b/pkg/kds/context/context.go index 982a777f88f9..7b866b566281 100644 --- a/pkg/kds/context/context.go +++ b/pkg/kds/context/context.go @@ -13,6 +13,7 @@ import ( "github.com/kumahq/kuma/pkg/kds/mux" "github.com/kumahq/kuma/pkg/kds/reconcile" "github.com/kumahq/kuma/pkg/kds/util" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" ) var log = core.Log.WithName("kds") @@ -42,9 +43,15 @@ func DefaultContext(manager manager.ResourceManager, zone string) *Context { // excludes Dataplanes and Ingresses from 'clusterID' cluster func GlobalProvidedFilter(rm manager.ResourceManager, configs map[string]bool) reconcile.ResourceFilter { return func(clusterID string, r model.Resource) bool { + if r.GetType() == mesh.ZoneIngressType { + return r.(*mesh.ZoneIngressResource).Spec.GetZone() != clusterID + } if r.GetType() == system.ConfigType && !configs[r.GetMeta().GetName()] { return false } + if r.GetType() == system.GlobalSecretType { + return zoneingress.IsSigningKeyResource(model.MetaToResourceKey(r.GetMeta())) + } if r.GetType() != mesh.DataplaneType { return true } @@ -72,6 +79,12 @@ func RemoteProvidedFilter(clusterName string) reconcile.ResourceFilter { if r.GetType() == mesh.DataplaneType { return clusterName == util.ZoneTag(r) } - return r.GetType() == mesh.DataplaneInsightType + if r.GetType() == mesh.DataplaneInsightType { + return true + } + if r.GetType() == mesh.ZoneIngressType && !r.(*mesh.ZoneIngressResource).IsRemoteIngress(clusterName) { + return true + } + return false } } diff --git a/pkg/kds/global/components.go b/pkg/kds/global/components.go index 3ae9341abff2..6043a34c921b 100644 --- a/pkg/kds/global/components.go +++ b/pkg/kds/global/components.go @@ -20,7 +20,7 @@ import ( kds_server "github.com/kumahq/kuma/pkg/kds/server" "github.com/kumahq/kuma/pkg/core" - "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/apis/system" "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/runtime" @@ -32,25 +32,28 @@ import ( var ( kdsGlobalLog = core.Log.WithName("kds-global") ProvidedTypes = []model.ResourceType{ - mesh.CircuitBreakerType, - mesh.DataplaneType, - mesh.ExternalServiceType, - mesh.FaultInjectionType, - mesh.HealthCheckType, - mesh.MeshType, - mesh.ProxyTemplateType, - mesh.RetryType, - mesh.TimeoutType, - mesh.TrafficLogType, - mesh.TrafficPermissionType, - mesh.TrafficRouteType, - mesh.TrafficTraceType, + core_mesh.CircuitBreakerType, + core_mesh.DataplaneType, + core_mesh.ZoneIngressType, + core_mesh.ExternalServiceType, + core_mesh.FaultInjectionType, + core_mesh.HealthCheckType, + core_mesh.MeshType, + core_mesh.ProxyTemplateType, + core_mesh.RetryType, + core_mesh.TimeoutType, + core_mesh.TrafficLogType, + core_mesh.TrafficPermissionType, + core_mesh.TrafficRouteType, + core_mesh.TrafficTraceType, system.SecretType, + system.GlobalSecretType, system.ConfigType, } ConsumedTypes = []model.ResourceType{ - mesh.DataplaneType, - mesh.DataplaneInsightType, + core_mesh.DataplaneType, + core_mesh.ZoneIngressType, + core_mesh.DataplaneInsightType, } ) @@ -121,6 +124,11 @@ func Callbacks(s sync_store.ResourceSyncer, k8sStore bool, kubeFactory resources util.AddSuffixToNames(rs.GetItems(), "default") } } + if rs.GetItemType() == core_mesh.ZoneIngressType { + for _, zi := range rs.(*core_mesh.ZoneIngressResourceList).Items { + zi.Spec.Zone = clusterName + } + } return s.Sync(rs, sync_store.PrefilterBy(func(r model.Resource) bool { return strings.HasPrefix(r.GetMeta().GetName(), fmt.Sprintf("%s.", clusterName)) }), sync_store.Zone(clusterName)) diff --git a/pkg/kds/remote/components.go b/pkg/kds/remote/components.go index 6f7d2339677e..49c05b727e28 100644 --- a/pkg/kds/remote/components.go +++ b/pkg/kds/remote/components.go @@ -3,6 +3,8 @@ package remote import ( "github.com/pkg/errors" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" + "github.com/kumahq/kuma/pkg/config/core/resources/store" "github.com/kumahq/kuma/pkg/core/runtime/component" "github.com/kumahq/kuma/pkg/kds/mux" @@ -24,12 +26,14 @@ import ( var ( kdsRemoteLog = core.Log.WithName("kds-remote") ProvidedTypes = []model.ResourceType{ + mesh.ZoneIngressType, mesh.DataplaneType, mesh.DataplaneInsightType, } ConsumedTypes = []model.ResourceType{ mesh.CircuitBreakerType, mesh.DataplaneType, + mesh.ZoneIngressType, mesh.ExternalServiceType, mesh.FaultInjectionType, mesh.HealthCheckType, @@ -42,6 +46,7 @@ var ( mesh.TrafficRouteType, mesh.TrafficTraceType, system.SecretType, + system.GlobalSecretType, system.ConfigType, } ) @@ -88,7 +93,7 @@ func Setup(rt core_runtime.Runtime) error { func Callbacks(rt core_runtime.Runtime, syncer sync_store.ResourceSyncer, k8sStore bool, localZone string, kubeFactory resources_k8s.KubeFactory) *kds_client.Callbacks { return &kds_client.Callbacks{ OnResourcesReceived: func(clusterID string, rs model.ResourceList) error { - if k8sStore && rs.GetItemType() != system.ConfigType && rs.GetItemType() != system.SecretType { + if k8sStore && rs.GetItemType() != system.ConfigType && rs.GetItemType() != system.SecretType && rs.GetItemType() != system.GlobalSecretType { // if type of Store is Kubernetes then we want to store upstream resources in dedicated Namespace. // KubernetesStore parses Name and considers substring after the last dot as a Namespace's Name. // System resources are not in the kubeFactory therefore we need explicit ifs for them @@ -105,11 +110,22 @@ func Callbacks(rt core_runtime.Runtime, syncer sync_store.ResourceSyncer, k8sSto return r.(*mesh.DataplaneResource).Spec.IsRemoteIngress(localZone) })) } + if rs.GetItemType() == mesh.ZoneIngressType { + core.Log.WithName("TEST").Info("OnResourcesReceived ZoneIngress", "resources", rs) + return syncer.Sync(rs, sync_store.PrefilterBy(func(r model.Resource) bool { + return r.(*mesh.ZoneIngressResource).IsRemoteIngress(localZone) + })) + } if rs.GetItemType() == system.ConfigType { return syncer.Sync(rs, sync_store.PrefilterBy(func(r model.Resource) bool { return rt.KDSContext().Configs[r.GetMeta().GetName()] })) } + if rs.GetItemType() == system.GlobalSecretType { + return syncer.Sync(rs, sync_store.PrefilterBy(func(r model.Resource) bool { + return r.GetMeta().GetName() == zoneingress.SigningKeyResourceKey().Name + })) + } return syncer.Sync(rs) }, } diff --git a/pkg/kds/types.go b/pkg/kds/types.go index d30f55aeba52..19b8d2fddc69 100644 --- a/pkg/kds/types.go +++ b/pkg/kds/types.go @@ -16,6 +16,7 @@ var ( SupportedTypes = []model.ResourceType{ mesh.CircuitBreakerType, mesh.DataplaneType, + mesh.ZoneIngressType, mesh.DataplaneInsightType, mesh.ExternalServiceType, mesh.FaultInjectionType, @@ -29,6 +30,7 @@ var ( mesh.TrafficRouteType, mesh.TrafficTraceType, system.SecretType, + system.GlobalSecretType, system.ConfigType, } ) diff --git a/pkg/plugins/bootstrap/k8s/xds/hooks/api_server_bypass.go b/pkg/plugins/bootstrap/k8s/xds/hooks/api_server_bypass.go index 5c2425e90664..96dc91f39690 100644 --- a/pkg/plugins/bootstrap/k8s/xds/hooks/api_server_bypass.go +++ b/pkg/plugins/bootstrap/k8s/xds/hooks/api_server_bypass.go @@ -32,6 +32,10 @@ func NewApiServerBypass(address string, port uint32) ApiServerBypass { } func (h ApiServerBypass) Modify(resources *core_xds.ResourceSet, ctx xds_context.Context, proxy *core_xds.Proxy) error { + if proxy.Dataplane == nil { + return nil + } + // backwards compatibility if proxy.Dataplane.Spec.IsIngress() || ctx.Mesh.Resource.Spec.IsPassthrough() { return nil } diff --git a/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types.go b/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types.go new file mode 100644 index 000000000000..56f46c409c58 --- /dev/null +++ b/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types.go @@ -0,0 +1,47 @@ +/* +Copyright 2019 Kuma authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ZoneIngress defines the desired state of ZoneIngress +type ZoneIngressSpec = map[string]interface{} + +// ZoneIngress is the Schema for the zone ingress API +type ZoneIngress struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Mesh string `json:"mesh,omitempty"` + + Spec ZoneIngressSpec `json:"spec,omitempty"` +} + +// ZoneIngressList contains a list of ZoneIngress +type ZoneIngressList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ZoneIngress `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ZoneIngress{}, &ZoneIngressList{}) +} diff --git a/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_deepcopy.go b/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_deepcopy.go new file mode 100644 index 000000000000..62c4eb4a8832 --- /dev/null +++ b/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_deepcopy.go @@ -0,0 +1,63 @@ +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (zi *ZoneIngress) DeepCopyInto(out *ZoneIngress) { + *out = *zi + out.TypeMeta = zi.TypeMeta + zi.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = runtime.DeepCopyJSON(zi.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficPermission. +func (zi *ZoneIngress) DeepCopy() *ZoneIngress { + if zi == nil { + return nil + } + out := new(ZoneIngress) + zi.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (zi *ZoneIngress) DeepCopyObject() runtime.Object { + if c := zi.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (l *ZoneIngressList) DeepCopyInto(out *ZoneIngressList) { + *out = *l + out.TypeMeta = l.TypeMeta + out.ListMeta = l.ListMeta + if l.Items != nil { + in, out := &l.Items, &out.Items + *out = make([]ZoneIngress, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficPermissionList. +func (l *ZoneIngressList) DeepCopy() *ZoneIngressList { + if l == nil { + return nil + } + out := new(ZoneIngressList) + l.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (l *ZoneIngressList) DeepCopyObject() runtime.Object { + if c := l.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_helpers.go b/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_helpers.go new file mode 100644 index 000000000000..628018aa28c5 --- /dev/null +++ b/pkg/plugins/resources/k8s/native/api/v1alpha1/zone_ingress_types_helpers.go @@ -0,0 +1,61 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + + "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/pkg/model" + "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/pkg/registry" +) + +func (zi *ZoneIngress) GetObjectMeta() *metav1.ObjectMeta { + return &zi.ObjectMeta +} + +func (zi *ZoneIngress) SetObjectMeta(m *metav1.ObjectMeta) { + zi.ObjectMeta = *m +} + +func (zi *ZoneIngress) GetMesh() string { + return zi.Mesh +} + +func (zi *ZoneIngress) SetMesh(mesh string) { + zi.Mesh = mesh +} + +func (zi *ZoneIngress) GetSpec() map[string]interface{} { + return zi.Spec +} + +func (zi *ZoneIngress) SetSpec(spec map[string]interface{}) { + zi.Spec = spec +} + +func (zi *ZoneIngress) Scope() model.Scope { + return model.ScopeNamespace +} + +func (l *ZoneIngressList) GetItems() []model.KubernetesObject { + result := make([]model.KubernetesObject, len(l.Items)) + for i := range l.Items { + result[i] = &l.Items[i] + } + return result +} + +func init() { + registry.RegisterObjectType(&proto.ZoneIngress{}, &ZoneIngress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: GroupVersion.String(), + Kind: "ZoneIngress", + }, + }) + registry.RegisterListType(&proto.ZoneIngress{}, &ZoneIngressList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: GroupVersion.String(), + Kind: "ZoneIngressList", + }, + }) +} diff --git a/pkg/plugins/resources/k8s/native/config/crd/bases/kuma.io_zoneingresses.yaml b/pkg/plugins/resources/k8s/native/config/crd/bases/kuma.io_zoneingresses.yaml new file mode 100644 index 000000000000..4ac80539c070 --- /dev/null +++ b/pkg/plugins/resources/k8s/native/config/crd/bases/kuma.io_zoneingresses.yaml @@ -0,0 +1,27 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object diff --git a/pkg/plugins/runtime/k8s/controllers/configmap_controller.go b/pkg/plugins/runtime/k8s/controllers/configmap_controller.go index 6bfc8c691f8d..73839a9fd3ef 100644 --- a/pkg/plugins/runtime/k8s/controllers/configmap_controller.go +++ b/pkg/plugins/runtime/k8s/controllers/configmap_controller.go @@ -85,6 +85,13 @@ func (r *ConfigMapReconciler) SetupWithManager(mgr kube_ctrl.Manager) error { ResourceConverter: r.ResourceConverter, }, }). + Watches(&kube_source.Kind{Type: &mesh_k8s.ZoneIngress{}}, &kube_handler.EnqueueRequestsFromMapFunc{ + ToRequests: &ZoneIngressToMeshMapper{ + Log: r.Log.WithName("zone-ingress-to-configmap-mapper"), + SystemNamespace: r.SystemNamespace, + ResourceConverter: r.ResourceConverter, + }, + }). Watches(&kube_source.Kind{Type: &mesh_k8s.ExternalService{}}, &kube_handler.EnqueueRequestsFromMapFunc{ ToRequests: &ExternalServiceToConfigMapsMapper{ Client: mgr.GetClient(), @@ -153,6 +160,7 @@ func (m *DataplaneToMeshMapper) Map(obj kube_handler.MapObject) []kube_reconile. return nil } + // backwards compatibility if dp.Spec.IsIngress() { meshSet := map[string]bool{} for _, service := range dp.Spec.GetNetworking().GetIngress().GetAvailableServices() { @@ -173,6 +181,38 @@ func (m *DataplaneToMeshMapper) Map(obj kube_handler.MapObject) []kube_reconile. }} } +type ZoneIngressToMeshMapper struct { + Log logr.Logger + ResourceConverter k8s_common.Converter + SystemNamespace string +} + +func (m *ZoneIngressToMeshMapper) Map(obj kube_handler.MapObject) []kube_reconile.Request { + cause, ok := obj.Object.(*mesh_k8s.ZoneIngress) + if !ok { + m.Log.WithValues("zoneIngress", obj.Meta).Error(errors.Errorf("wrong argument type: expected %T, got %T", cause, obj.Object), "wrong argument type") + return nil + } + zoneIngress := core_mesh.NewZoneIngressResource() + if err := m.ResourceConverter.ToCoreResource(cause, zoneIngress); err != nil { + converterLog.Error(err, "failed to parse ZoneIngress", "zoneIngress", cause.Spec) + return nil + } + + meshSet := map[string]bool{} + for _, service := range zoneIngress.Spec.GetAvailableServices() { + meshSet[service.Mesh] = true + } + + var requests []kube_reconile.Request + for mesh := range meshSet { + requests = append(requests, kube_reconile.Request{ + NamespacedName: kube_types.NamespacedName{Namespace: m.SystemNamespace, Name: vips.ConfigKey(mesh)}, + }) + } + return requests +} + type ExternalServiceToConfigMapsMapper struct { kube_client.Client Log logr.Logger diff --git a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go index 4b2b3d72aee9..e4597e117652 100644 --- a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go @@ -7,6 +7,7 @@ import ( kube_core "k8s.io/api/core/v1" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata" ) @@ -21,7 +22,7 @@ var NodePortAddressPriority = []kube_core.NodeAddressType{ kube_core.NodeInternalIP, } -func (p *PodConverter) IngressFor(dp *mesh_proto.Dataplane, pod *kube_core.Pod, services []*kube_core.Service) error { +func (p *PodConverter) IngressFor(zoneIngress *mesh_proto.ZoneIngress, pod *kube_core.Pod, services []*kube_core.Service) error { if len(services) != 1 { return errors.Errorf("ingress should be matched by exactly one service. Matched %d services", len(services)) } @@ -32,15 +33,11 @@ func (p *PodConverter) IngressFor(dp *mesh_proto.Dataplane, pod *kube_core.Pod, if len(ifaces) != 1 { return errors.Errorf("generated %d inbound interfaces, expected 1. Interfaces: %v", len(ifaces), ifaces) } - if dp.Networking == nil { - dp.Networking = &mesh_proto.Dataplane_Networking{ - Ingress: &mesh_proto.Dataplane_Networking_Ingress{}, - } - } - dp.Networking.Inbound = ifaces - dp.Networking.Address = pod.Status.PodIP - coords, err := p.coordinatesFromAnnotations(metadata.Annotations(pod.Annotations)) + zoneIngress.Address = pod.Status.PodIP + zoneIngress.Port = ifaces[0].Port + + coords, err := p.coordinatesFromAnnotations(pod.Annotations) if err != nil { return err } @@ -53,8 +50,8 @@ func (p *PodConverter) IngressFor(dp *mesh_proto.Dataplane, pod *kube_core.Pod, } if coords != nil { - dp.Networking.Ingress.PublicAddress = coords.address - dp.Networking.Ingress.PublicPort = coords.port + zoneIngress.AdvertisedAddress = coords.address + zoneIngress.AdvertisedPort = coords.port } return nil } diff --git a/pkg/plugins/runtime/k8s/controllers/outbound_converter.go b/pkg/plugins/runtime/k8s/controllers/outbound_converter.go index 8dd069d73b25..0eb7732c0645 100644 --- a/pkg/plugins/runtime/k8s/controllers/outbound_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/outbound_converter.go @@ -22,10 +22,12 @@ import ( func (p *PodConverter) OutboundInterfacesFor( pod *kube_core.Pod, others []*mesh_k8s.Dataplane, + zoneIngresses []*mesh_k8s.ZoneIngress, externalServices []*mesh_k8s.ExternalService, vips vips.List, ) ([]*mesh_proto.Dataplane_Networking_Outbound, error) { var outbounds []*mesh_proto.Dataplane_Networking_Outbound + dataplanes := []*core_mesh.DataplaneResource{} for _, other := range others { dp := core_mesh.NewDataplaneResource() @@ -35,6 +37,7 @@ func (p *PodConverter) OutboundInterfacesFor( } dataplanes = append(dataplanes, dp) } + externalServicesRes := []*core_mesh.ExternalServiceResource{} for _, es := range externalServices { res := core_mesh.NewExternalServiceResource() @@ -45,6 +48,16 @@ func (p *PodConverter) OutboundInterfacesFor( externalServicesRes = append(externalServicesRes, res) } + zoneIngressesRes := []*core_mesh.ZoneIngressResource{} + for _, zi := range zoneIngresses { + res := core_mesh.NewZoneIngressResource() + if err := p.ResourceConverter.ToCoreResource(zi, res); err != nil { + converterLog.Error(err, "failed to parse ZoneIngress", "zoneIngress", zi.Spec) + continue + } + zoneIngressesRes = append(zoneIngressesRes, res) + } + endpoints := endpointsByService(dataplanes) for _, serviceTag := range endpoints.Services() { service, port, err := p.k8sService(serviceTag) @@ -89,7 +102,7 @@ func (p *PodConverter) OutboundInterfacesFor( Mesh: MeshFor(pod), Name: pod.Name, } - outbounds = append(outbounds, dns.VIPOutbounds(resourceKey, dataplanes, vips, externalServicesRes)...) + outbounds = append(outbounds, dns.VIPOutbounds(resourceKey, dataplanes, zoneIngressesRes, vips, externalServicesRes)...) return outbounds, nil } diff --git a/pkg/plugins/runtime/k8s/controllers/pod_controller.go b/pkg/plugins/runtime/k8s/controllers/pod_controller.go index e58dd96b4172..5ef191c0dd66 100644 --- a/pkg/plugins/runtime/k8s/controllers/pod_controller.go +++ b/pkg/plugins/runtime/k8s/controllers/pod_controller.go @@ -120,6 +120,11 @@ func (r *PodReconciler) Reconcile(req kube_ctrl.Request) (kube_ctrl.Result, erro return kube_ctrl.Result{}, err } + zoneIngresses, err := r.findZoneIngresses(pod) + if err != nil { + return kube_ctrl.Result{}, err + } + r.Log.WithValues("req", req).V(1).Info("other dataplanes", "others", others) vips, err := r.Persistence.GetByMesh(MeshFor(pod)) @@ -127,7 +132,7 @@ func (r *PodReconciler) Reconcile(req kube_ctrl.Request) (kube_ctrl.Result, erro return kube_ctrl.Result{}, err } - if err := r.createOrUpdateDataplane(pod, services, externalServices, others, vips); err != nil { + if err := r.createOrUpdateDataplane(pod, services, externalServices, others, zoneIngresses, vips); err != nil { return kube_ctrl.Result{}, err } @@ -202,11 +207,32 @@ func (r *PodReconciler) findOtherDataplanes(pod *kube_core.Pod) ([]*mesh_k8s.Dat return otherDataplanes, nil } +func (r *PodReconciler) findZoneIngresses(pod *kube_core.Pod) ([]*mesh_k8s.ZoneIngress, error) { + ctx := context.Background() + + // List all ZoneIngresses + zoneIngresses := &mesh_k8s.ZoneIngressList{} + if err := r.List(ctx, zoneIngresses); err != nil { + log := r.Log.WithValues("pod", kube_types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}) + log.Error(err, "unable to list ZoneIngresses") + return nil, err + } + + rv := []*mesh_k8s.ZoneIngress{} + for i := range zoneIngresses.Items { + zi := zoneIngresses.Items[i] + rv = append(rv, &zi) + } + + return rv, nil +} + func (r *PodReconciler) createOrUpdateDataplane( pod *kube_core.Pod, services []*kube_core.Service, externalServices []*mesh_k8s.ExternalService, others []*mesh_k8s.Dataplane, + zoneIngresses []*mesh_k8s.ZoneIngress, vips vips.List, ) error { ctx := context.Background() @@ -218,7 +244,7 @@ func (r *PodReconciler) createOrUpdateDataplane( }, } operationResult, err := kube_controllerutil.CreateOrUpdate(ctx, r.Client, dataplane, func() error { - if err := r.PodConverter.PodToDataplane(dataplane, pod, services, externalServices, others, vips); err != nil { + if err := r.PodConverter.PodToDataplane(dataplane, pod, services, externalServices, others, zoneIngresses, vips); err != nil { return errors.Wrap(err, "unable to translate a Pod into a Dataplane") } if err := kube_controllerutil.SetControllerReference(pod, dataplane, r.Scheme); err != nil { @@ -244,12 +270,12 @@ func (r *PodReconciler) createOrUpdateDataplane( func (r *PodReconciler) createOrUpdateIngress(pod *kube_core.Pod, services []*kube_core.Service) error { ctx := context.Background() - ingress := &mesh_k8s.Dataplane{ + ingress := &mesh_k8s.ZoneIngress{ ObjectMeta: kube_meta.ObjectMeta{ Namespace: pod.Namespace, Name: pod.Name, }, - Mesh: model.DefaultMesh, + Mesh: model.NoMesh, } operationResult, err := kube_controllerutil.CreateOrUpdate(ctx, r.Client, ingress, func() error { if err := r.PodConverter.PodToIngress(ingress, pod, services); err != nil { diff --git a/pkg/plugins/runtime/k8s/controllers/pod_converter.go b/pkg/plugins/runtime/k8s/controllers/pod_converter.go index 15c5038cafd1..7cb713a894cc 100644 --- a/pkg/plugins/runtime/k8s/controllers/pod_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/pod_converter.go @@ -37,10 +37,11 @@ func (p *PodConverter) PodToDataplane( services []*kube_core.Service, externalServices []*mesh_k8s.ExternalService, others []*mesh_k8s.Dataplane, + zoneIngresses []*mesh_k8s.ZoneIngress, vips vips.List, ) error { dataplane.Mesh = MeshFor(pod) - dataplaneProto, err := p.DataplaneFor(pod, services, externalServices, others, vips) + dataplaneProto, err := p.DataplaneFor(pod, services, externalServices, others, zoneIngresses, vips) if err != nil { return err } @@ -52,21 +53,20 @@ func (p *PodConverter) PodToDataplane( return nil } -func (p *PodConverter) PodToIngress(dataplane *mesh_k8s.Dataplane, pod *kube_core.Pod, services []*kube_core.Service) error { - dataplane.Mesh = MeshFor(pod) - dataplaneProto := &mesh_proto.Dataplane{} - if err := util_proto.FromMap(dataplane.Spec, dataplaneProto); err != nil { +func (p *PodConverter) PodToIngress(zoneIngress *mesh_k8s.ZoneIngress, pod *kube_core.Pod, services []*kube_core.Service) error { + zoneIngressProto := &mesh_proto.ZoneIngress{} + if err := util_proto.FromMap(zoneIngress.Spec, zoneIngressProto); err != nil { return err } // Pass the current dataplane so we won't override available services in Ingress section - if err := p.IngressFor(dataplaneProto, pod, services); err != nil { + if err := p.IngressFor(zoneIngressProto, pod, services); err != nil { return err } - spec, err := util_proto.ToMap(dataplaneProto) + spec, err := util_proto.ToMap(zoneIngressProto) if err != nil { return err } - dataplane.Spec = spec + zoneIngress.Spec = spec return nil } @@ -83,6 +83,7 @@ func (p *PodConverter) DataplaneFor( services []*kube_core.Service, externalServices []*mesh_k8s.ExternalService, others []*mesh_k8s.Dataplane, + zoneIngresses []*mesh_k8s.ZoneIngress, vips vips.List, ) (*mesh_proto.Dataplane, error) { dataplane := &mesh_proto.Dataplane{ @@ -144,7 +145,7 @@ func (p *PodConverter) DataplaneFor( dataplane.Networking.Inbound = ifaces } - ofaces, err := p.OutboundInterfacesFor(pod, others, externalServices, vips) + ofaces, err := p.OutboundInterfacesFor(pod, others, zoneIngresses, externalServices, vips) if err != nil { return nil, err } diff --git a/pkg/plugins/runtime/k8s/controllers/pod_converter_test.go b/pkg/plugins/runtime/k8s/controllers/pod_converter_test.go index bdbc90438403..4385c7681c38 100644 --- a/pkg/plugins/runtime/k8s/controllers/pod_converter_test.go +++ b/pkg/plugins/runtime/k8s/controllers/pod_converter_test.go @@ -7,14 +7,15 @@ import ( "io/ioutil" "path/filepath" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - "github.com/kumahq/kuma/pkg/dns/vips" - "github.com/kumahq/kuma/pkg/plugins/resources/k8s" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/dns/vips" + "github.com/kumahq/kuma/pkg/plugins/resources/k8s" + . "github.com/kumahq/kuma/pkg/test/matchers" + "github.com/ghodss/yaml" "github.com/pkg/errors" @@ -139,7 +140,7 @@ var _ = Describe("PodToDataplane(..)", func() { // when dataplane := &mesh_k8s.Dataplane{} - err = converter.PodToDataplane(dataplane, pod, services, []*mesh_k8s.ExternalService{}, otherDataplanes, vips.List{}) + err = converter.PodToDataplane(dataplane, pod, services, []*mesh_k8s.ExternalService{}, otherDataplanes, nil, vips.List{}) // then Expect(err).ToNot(HaveOccurred()) @@ -270,17 +271,15 @@ var _ = Describe("PodToDataplane(..)", func() { } // when - dataplane := &mesh_k8s.Dataplane{} - err = converter.PodToIngress(dataplane, pod, services) + ingress := &mesh_k8s.ZoneIngress{} + err = converter.PodToIngress(ingress, pod, services) // then Expect(err).ToNot(HaveOccurred()) - actual, err := json.Marshal(dataplane) - Expect(err).ToNot(HaveOccurred()) - expected, err := ioutil.ReadFile(filepath.Join("testdata", "ingress", given.dataplane)) + actual, err := yaml.Marshal(ingress) Expect(err).ToNot(HaveOccurred()) - Expect(actual).To(MatchYAML(expected)) + Expect(actual).To(MatchGoldenYAML(filepath.Join("testdata", "ingress", given.dataplane))) }, Entry("01. Ingress with load balancer service and hostname", testCase{ // AWS use case pod: "01.pod.yaml", @@ -338,7 +337,7 @@ var _ = Describe("PodToDataplane(..)", func() { dataplane := &mesh_k8s.Dataplane{} // when - err = converter.PodToDataplane(dataplane, pod, services, []*mesh_k8s.ExternalService{}, nil, vips.List{}) + err = converter.PodToDataplane(dataplane, pod, services, []*mesh_k8s.ExternalService{}, nil, nil, vips.List{}) // then Expect(err).To(HaveOccurred()) diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml index cdd6bbf2a357..ae24421c747b 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml @@ -1,16 +1,7 @@ -mesh: default metadata: creationTimestamp: null spec: - networking: - address: 192.168.0.1 - inbound: - - port: 10001 - tags: - app: kuma-ingress - kuma.io/protocol: tcp - kuma.io/service: kuma-ingress_kuma-system_svc_10001 - kuma.io/zone: zone-1 - ingress: - publicAddress: kuma-ingress.com - publicPort: 10001 + address: 192.168.0.1 + advertisedAddress: kuma-ingress.com + advertisedPort: 10001 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml index ef8199ed827b..2145f8335e80 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml @@ -1,16 +1,7 @@ -mesh: default metadata: creationTimestamp: null spec: - networking: - address: 192.168.0.1 - inbound: - - port: 10001 - tags: - app: kuma-ingress - kuma.io/protocol: tcp - kuma.io/service: kuma-ingress_kuma-system_svc_10001 - kuma.io/zone: zone-1 - ingress: - publicAddress: 192.168.100.1 - publicPort: 10001 + address: 192.168.0.1 + advertisedAddress: 192.168.100.1 + advertisedPort: 10001 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml index 91aa67cb5e2c..eb249041edb5 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml @@ -1,14 +1,5 @@ -mesh: default metadata: creationTimestamp: null spec: - networking: - address: 192.168.0.1 - inbound: - - port: 10001 - tags: - app: kuma-ingress - kuma.io/protocol: tcp - kuma.io/service: kuma-ingress_kuma-system_svc_10001 - kuma.io/zone: zone-1 - ingress: {} + address: 192.168.0.1 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml index 304ef417e9b9..73509704d540 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml @@ -1,16 +1,7 @@ -mesh: default metadata: creationTimestamp: null spec: - networking: - address: 192.168.0.1 - inbound: - - port: 10001 - tags: - app: kuma-ingress - kuma.io/protocol: tcp - kuma.io/service: kuma-ingress_kuma-system_svc_10001 - kuma.io/zone: zone-1 - ingress: - publicAddress: 34.72.129.131 - publicPort: 12345 + address: 192.168.0.1 + advertisedAddress: 34.72.129.131 + advertisedPort: 12345 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml index 046ed3795378..e6f1c123af54 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml @@ -1,16 +1,7 @@ -mesh: default metadata: creationTimestamp: null spec: - networking: - address: 192.168.0.1 - inbound: - - port: 10001 - tags: - app: kuma-ingress - kuma.io/protocol: tcp - kuma.io/service: kuma-ingress_kuma-system_svc_10001 - kuma.io/zone: zone-1 - ingress: - publicAddress: 10.128.15.193 - publicPort: 12345 + address: 192.168.0.1 + advertisedAddress: 10.128.15.193 + advertisedPort: 12345 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml index e2c72a7da8ca..a2988744b5fe 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml @@ -1,16 +1,7 @@ -mesh: default metadata: creationTimestamp: null spec: - networking: - address: 192.168.0.1 - inbound: - - port: 10001 - tags: - app: kuma-ingress - kuma.io/protocol: tcp - kuma.io/service: kuma-ingress_kuma-system_svc_10001 - kuma.io/zone: zone-1 - ingress: - publicAddress: custom-address.com - publicPort: 1234 + address: 192.168.0.1 + advertisedAddress: custom-address.com + advertisedPort: 1234 + port: 10001 diff --git a/pkg/plugins/runtime/universal/outbound/outbound.go b/pkg/plugins/runtime/universal/outbound/outbound.go index f8f87761975f..0db641f3294a 100644 --- a/pkg/plugins/runtime/universal/outbound/outbound.go +++ b/pkg/plugins/runtime/universal/outbound/outbound.go @@ -12,7 +12,7 @@ import ( "github.com/golang/protobuf/proto" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" "github.com/kumahq/kuma/pkg/core/resources/store" "github.com/kumahq/kuma/pkg/dns" @@ -62,41 +62,48 @@ func (v *VIPOutboundsReconciler) Start(stop <-chan struct{}) error { func (v *VIPOutboundsReconciler) UpdateVIPOutbounds(ctx context.Context) error { // First get all ingresses - var ingresses []*mesh.DataplaneResource - dpList := &mesh.DataplaneResourceList{} + // backward compatibility + var dpIngresses []*core_mesh.DataplaneResource + dpList := &core_mesh.DataplaneResourceList{} if err := v.rorm.List(ctx, dpList); err != nil { return err } for _, dp := range dpList.Items { if dp.Spec.IsIngress() { - ingresses = append(ingresses, dp) + dpIngresses = append(dpIngresses, dp) } } + + zoneIngresses := &core_mesh.ZoneIngressResourceList{} + if err := v.rorm.List(ctx, zoneIngresses); err != nil { + return err + } + // Then add outbounds to each Dataplane - meshes := &mesh.MeshResourceList{} + meshes := &core_mesh.MeshResourceList{} if err := v.rorm.List(ctx, meshes); err != nil { return err } for _, m := range meshes.Items { - dpList := &mesh.DataplaneResourceList{} + dpList := &core_mesh.DataplaneResourceList{} if err := v.rorm.List(ctx, dpList, store.ListByMesh(m.Meta.GetName())); err != nil { return err } - externalServices := &mesh.ExternalServiceResourceList{} + externalServices := &core_mesh.ExternalServiceResourceList{} if err := v.rorm.List(ctx, externalServices, store.ListByMesh(m.Meta.GetName())); err != nil { return err } dpsUpdated := 0 - allDps := make([]*mesh.DataplaneResource, len(ingresses)+len(dpList.Items)) - copy(allDps[:len(ingresses)], ingresses) - copy(allDps[len(ingresses):], dpList.Items) + allDps := make([]*core_mesh.DataplaneResource, len(dpIngresses)+len(dpList.Items)) + copy(allDps[:len(dpIngresses)], dpIngresses) + copy(allDps[len(dpIngresses):], dpList.Items) for _, dp := range dpList.Items { if dp.Spec.Networking.GetTransparentProxying() == nil || dp.Spec.IsIngress() { continue } - newOutbounds := dns.VIPOutbounds(model.MetaToResourceKey(dp.Meta), allDps, v.resolver.GetVIPs(), externalServices.Items) + newOutbounds := dns.VIPOutbounds(model.MetaToResourceKey(dp.Meta), allDps, zoneIngresses.Items, v.resolver.GetVIPs(), externalServices.Items) if outboundsEqual(newOutbounds, dp.Spec.Networking.Outbound) { continue diff --git a/pkg/sds/server/v2/reconciler.go b/pkg/sds/server/v2/reconciler.go index 5b7d428d8076..7bf0a89c8326 100644 --- a/pkg/sds/server/v2/reconciler.go +++ b/pkg/sds/server/v2/reconciler.go @@ -59,14 +59,14 @@ type snapshotInfo struct { generation time.Time } -func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey) error { - proxyID := core_xds.FromResourceKey(dataplaneId).String() +func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey, proxyType mesh_proto.DpType) error { + proxyID := core_xds.FromResourceKey(proxyType, dataplaneId).String() dataplane := mesh_core.NewDataplaneResource() if err := d.readOnlyResManager.Get(context.Background(), dataplane, core_store.GetBy(dataplaneId)); err != nil { if core_store.IsResourceNotFound(err) { sdsServerLog.V(1).Info("Dataplane not found. Clearing the Snapshot.", "dataplaneId", dataplaneId) - if err := d.Cleanup(dataplaneId); err != nil { + if err := d.Cleanup(dataplaneId, proxyType); err != nil { return errors.Wrap(err, "could not cleanup snapshot") } return nil @@ -81,7 +81,7 @@ func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey) erro if !mesh.MTLSEnabled() { sdsServerLog.V(1).Info("mTLS for Mesh disabled. Clearing the Snapshot.", "dataplaneId", dataplaneId) - if err := d.Cleanup(dataplaneId); err != nil { + if err := d.Cleanup(dataplaneId, proxyType); err != nil { return errors.Wrap(err, "could not cleanup snapshot") } return nil @@ -113,8 +113,8 @@ func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey) erro return nil } -func (d *DataplaneReconciler) Cleanup(dataplaneId core_model.ResourceKey) error { - proxyID := core_xds.FromResourceKey(dataplaneId).String() +func (d *DataplaneReconciler) Cleanup(dataplaneId core_model.ResourceKey, proxyType mesh_proto.DpType) error { + proxyID := core_xds.FromResourceKey(proxyType, dataplaneId).String() if err := d.cache.SetSnapshot(proxyID, envoy_cache.Snapshot{}); err != nil { return err } diff --git a/pkg/sds/server/v2/server.go b/pkg/sds/server/v2/server.go index c470b16c9e19..1c83bfa4bcc2 100644 --- a/pkg/sds/server/v2/server.go +++ b/pkg/sds/server/v2/server.go @@ -10,6 +10,8 @@ import ( envoy_server "github.com/envoyproxy/go-control-plane/pkg/server/v2" "github.com/go-logr/logr" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_runtime "github.com/kumahq/kuma/pkg/core/runtime" @@ -73,7 +75,7 @@ func RegisterSDS(rt core_runtime.Runtime, sdsMetrics *sds_metrics.Metrics) error } func syncTracker(reconciler *DataplaneReconciler, refresh time.Duration, sdsMetrics *sds_metrics.Metrics) (util_xds.Callbacks, error) { - return xds_callbacks.NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64) util_watchdog.Watchdog { + return xds_callbacks.NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { return &util_watchdog.SimpleWatchdog{ NewTicker: func() *time.Ticker { sdsMetrics.IncrementActiveWatchdogs(envoy_common.APIV2) @@ -84,14 +86,14 @@ func syncTracker(reconciler *DataplaneReconciler, refresh time.Duration, sdsMetr defer func() { sdsMetrics.SdsGeneration(envoy_common.APIV2).Observe(float64(core.Now().Sub(start).Milliseconds())) }() - return reconciler.Reconcile(dataplaneId) + return reconciler.Reconcile(dataplaneId, proxyType) }, OnError: func(err error) { sdsMetrics.SdsGenerationsErrors(envoy_common.APIV2).Inc() sdsServerLog.Error(err, "OnTick() failed") }, OnStop: func() { - if err := reconciler.Cleanup(dataplaneId); err != nil { + if err := reconciler.Cleanup(dataplaneId, proxyType); err != nil { sdsServerLog.Error(err, "could not cleanup sync", "dataplane", dataplaneId) } sdsMetrics.DecrementActiveWatchdogs(envoy_common.APIV2) diff --git a/pkg/sds/server/v3/reconciler.go b/pkg/sds/server/v3/reconciler.go index 868f1f35b486..2e9d55387b37 100644 --- a/pkg/sds/server/v3/reconciler.go +++ b/pkg/sds/server/v3/reconciler.go @@ -59,14 +59,14 @@ type snapshotInfo struct { generation time.Time } -func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey) error { - proxyID := core_xds.FromResourceKey(dataplaneId).String() +func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey, proxyType mesh_proto.DpType) error { + proxyID := core_xds.FromResourceKey(proxyType, dataplaneId).String() dataplane := mesh_core.NewDataplaneResource() if err := d.readOnlyResManager.Get(context.Background(), dataplane, core_store.GetBy(dataplaneId)); err != nil { if core_store.IsResourceNotFound(err) { sdsServerLog.V(1).Info("Dataplane not found. Clearing the Snapshot.", "dataplaneId", dataplaneId) - if err := d.Cleanup(dataplaneId); err != nil { + if err := d.Cleanup(dataplaneId, proxyType); err != nil { return errors.Wrap(err, "could not cleanup snapshot") } return nil @@ -81,7 +81,7 @@ func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey) erro if !mesh.MTLSEnabled() { sdsServerLog.V(1).Info("mTLS for Mesh disabled. Clearing the Snapshot.", "dataplaneId", dataplaneId) - if err := d.Cleanup(dataplaneId); err != nil { + if err := d.Cleanup(dataplaneId, proxyType); err != nil { return errors.Wrap(err, "could not cleanup snapshot") } return nil @@ -113,8 +113,8 @@ func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey) erro return nil } -func (d *DataplaneReconciler) Cleanup(dataplaneId core_model.ResourceKey) error { - proxyID := core_xds.FromResourceKey(dataplaneId).String() +func (d *DataplaneReconciler) Cleanup(dataplaneId core_model.ResourceKey, proxyType mesh_proto.DpType) error { + proxyID := core_xds.FromResourceKey(proxyType, dataplaneId).String() if err := d.cache.SetSnapshot(proxyID, envoy_cache.Snapshot{}); err != nil { return err } diff --git a/pkg/sds/server/v3/server.go b/pkg/sds/server/v3/server.go index c1a29c2d32a5..1685185838e0 100644 --- a/pkg/sds/server/v3/server.go +++ b/pkg/sds/server/v3/server.go @@ -10,6 +10,8 @@ import ( envoy_server "github.com/envoyproxy/go-control-plane/pkg/server/v3" "github.com/go-logr/logr" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_runtime "github.com/kumahq/kuma/pkg/core/runtime" @@ -73,7 +75,7 @@ func RegisterSDS(rt core_runtime.Runtime, sdsMetrics *sds_metrics.Metrics) error } func syncTracker(reconciler *DataplaneReconciler, refresh time.Duration, sdsMetrics *sds_metrics.Metrics) (util_xds.Callbacks, error) { - return xds_callbacks.NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64) util_watchdog.Watchdog { + return xds_callbacks.NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { return &util_watchdog.SimpleWatchdog{ NewTicker: func() *time.Ticker { sdsMetrics.IncrementActiveWatchdogs(envoy_common.APIV3) @@ -84,14 +86,14 @@ func syncTracker(reconciler *DataplaneReconciler, refresh time.Duration, sdsMetr defer func() { sdsMetrics.SdsGeneration(envoy_common.APIV3).Observe(float64(core.Now().Sub(start).Milliseconds())) }() - return reconciler.Reconcile(dataplaneId) + return reconciler.Reconcile(dataplaneId, proxyType) }, OnError: func(err error) { sdsMetrics.SdsGenerationsErrors(envoy_common.APIV3).Inc() sdsServerLog.Error(err, "OnTick() failed") }, OnStop: func() { - if err := reconciler.Cleanup(dataplaneId); err != nil { + if err := reconciler.Cleanup(dataplaneId, proxyType); err != nil { sdsServerLog.Error(err, "could not cleanup sync", "dataplane", dataplaneId) } sdsMetrics.DecrementActiveWatchdogs(envoy_common.APIV3) diff --git a/pkg/tokens/builtin/components.go b/pkg/tokens/builtin/components.go index ea570bba9d5c..a96c02c3c75b 100644 --- a/pkg/tokens/builtin/components.go +++ b/pkg/tokens/builtin/components.go @@ -3,6 +3,7 @@ package builtin import ( "github.com/kumahq/kuma/pkg/core/resources/manager" "github.com/kumahq/kuma/pkg/tokens/builtin/issuer" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" ) func NewDataplaneTokenIssuer(resManager manager.ReadOnlyResourceManager) (issuer.DataplaneTokenIssuer, error) { @@ -10,3 +11,9 @@ func NewDataplaneTokenIssuer(resManager manager.ReadOnlyResourceManager) (issuer return issuer.GetSigningKey(resManager, issuer.DataplaneTokenPrefix, meshName) }), nil } + +func NewZoneIngressTokenIssuer(resManager manager.ReadOnlyResourceManager) (zoneingress.TokenIssuer, error) { + return zoneingress.NewTokenIssuer(func() ([]byte, error) { + return zoneingress.GetSigningKey(resManager) + }), nil +} diff --git a/pkg/tokens/builtin/server/types/zoneingress_token_request.go b/pkg/tokens/builtin/server/types/zoneingress_token_request.go new file mode 100644 index 000000000000..0b18f65cfcb6 --- /dev/null +++ b/pkg/tokens/builtin/server/types/zoneingress_token_request.go @@ -0,0 +1,5 @@ +package types + +type ZoneIngressTokenRequest struct { + Name string `json:"name"` +} diff --git a/pkg/tokens/builtin/server/webservice.go b/pkg/tokens/builtin/server/webservice.go index 7eeee9e3afcf..6cb0df5a3d39 100644 --- a/pkg/tokens/builtin/server/webservice.go +++ b/pkg/tokens/builtin/server/webservice.go @@ -5,6 +5,8 @@ import ( "github.com/emicklei/go-restful" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" "github.com/kumahq/kuma/pkg/core/rest/errors" @@ -15,27 +17,31 @@ import ( var log = core.Log.WithName("dataplane-token-ws") -type dataplaneTokenWebService struct { - issuer issuer.DataplaneTokenIssuer +type tokenWebService struct { + issuer issuer.DataplaneTokenIssuer + zoneIngressIssuer zoneingress.TokenIssuer } -func NewWebservice(issuer issuer.DataplaneTokenIssuer) *restful.WebService { - ws := dataplaneTokenWebService{ - issuer: issuer, +func NewWebservice(issuer issuer.DataplaneTokenIssuer, zoneIngressIssuer zoneingress.TokenIssuer) *restful.WebService { + ws := tokenWebService{ + issuer: issuer, + zoneIngressIssuer: zoneIngressIssuer, } return ws.createWs() } -func (d *dataplaneTokenWebService) createWs() *restful.WebService { +func (d *tokenWebService) createWs() *restful.WebService { ws := new(restful.WebService). Consumes(restful.MIME_JSON). Produces(restful.MIME_JSON) ws.Path("/tokens"). - Route(ws.POST("").To(d.handleIdentityRequest)) + Route(ws.POST("").To(d.handleIdentityRequest)). // backwards compatibility + Route(ws.POST("/dataplane").To(d.handleIdentityRequest)). + Route(ws.POST("/zone-ingress").To(d.handleZoneIngressIdentityRequest)) return ws } -func (d *dataplaneTokenWebService) handleIdentityRequest(request *restful.Request, response *restful.Response) { +func (d *tokenWebService) handleIdentityRequest(request *restful.Request, response *restful.Response) { idReq := types.DataplaneTokenRequest{} if err := request.ReadEntity(&idReq); err != nil { log.Error(err, "Could not read a request") @@ -63,6 +69,35 @@ func (d *dataplaneTokenWebService) handleIdentityRequest(request *restful.Reques response.Header().Set("content-type", "text/plain") if _, err := response.Write([]byte(token)); err != nil { - log.Error(err, "Could write a response") + log.Error(err, "Could not write a response") + } +} + +func (d *tokenWebService) handleZoneIngressIdentityRequest(request *restful.Request, response *restful.Response) { + idReq := types.ZoneIngressTokenRequest{} + if err := request.ReadEntity(&idReq); err != nil { + log.Error(err, "Could not read a request") + response.WriteHeader(http.StatusBadRequest) + return + } + + if idReq.Name == "" { + verr := validators.ValidationError{} + verr.AddViolation("name", "cannot be empty") + errors.HandleError(response, verr.OrNil(), "Invalid request") + return + } + + token, err := d.zoneIngressIssuer.Generate(zoneingress.Identity{ + Name: idReq.Name, + }) + if err != nil { + errors.HandleError(response, err, "Could not issue a token") + return + } + + response.Header().Set("content-type", "text/plain") + if _, err := response.Write([]byte(token)); err != nil { + log.Error(err, "Could not write a response") } } diff --git a/pkg/tokens/builtin/server/webservice_test.go b/pkg/tokens/builtin/server/webservice_test.go index 36dd99d31c38..2ff6b57a5fe0 100644 --- a/pkg/tokens/builtin/server/webservice_test.go +++ b/pkg/tokens/builtin/server/webservice_test.go @@ -15,6 +15,8 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" + "github.com/kumahq/kuma/pkg/tokens/builtin/issuer" "github.com/kumahq/kuma/pkg/tokens/builtin/server" "github.com/kumahq/kuma/pkg/tokens/builtin/server/types" @@ -34,13 +36,26 @@ func (s *staticTokenIssuer) Validate(token issuer.Token, meshName string) (issue return issuer.DataplaneIdentity{}, errors.New("not implemented") } +type zoneIngressStaticTokenIssuer struct { +} + +var _ zoneingress.TokenIssuer = &zoneIngressStaticTokenIssuer{} + +func (z *zoneIngressStaticTokenIssuer) Generate(identity zoneingress.Identity) (zoneingress.Token, error) { + return fmt.Sprintf("token-for-%s", identity.Name), nil +} + +func (z *zoneIngressStaticTokenIssuer) Validate(token zoneingress.Token) (zoneingress.Identity, error) { + return zoneingress.Identity{}, errors.New("not implemented") +} + var _ = Describe("Dataplane Token Webservice", func() { const credentials = "test" var url string BeforeEach(func() { - ws := server.NewWebservice(&staticTokenIssuer{credentials}) + ws := server.NewWebservice(&staticTokenIssuer{credentials}, &zoneIngressStaticTokenIssuer{}) container := restful.NewContainer() container.Add(ws) diff --git a/pkg/tokens/builtin/zoneingress/issuer.go b/pkg/tokens/builtin/zoneingress/issuer.go new file mode 100644 index 000000000000..cd3ec196670d --- /dev/null +++ b/pkg/tokens/builtin/zoneingress/issuer.go @@ -0,0 +1,91 @@ +package zoneingress + +import ( + "github.com/dgrijalva/jwt-go" + "github.com/pkg/errors" +) + +type Token = string + +type Identity struct { + Name string +} + +// TokenIssuer issues Zone Ingress Tokens used then for proving identity of the zone ingresses. +// Issued token can be bound by zone name. +// See pkg/sds/auth/universal/authenticator.go to check algorithm for authentication +type TokenIssuer interface { + Generate(identity Identity) (Token, error) + Validate(token Token) (Identity, error) +} + +type claims struct { + Name string + jwt.StandardClaims +} + +type SigningKeyAccessor func() ([]byte, error) + +var _ TokenIssuer = &jwtTokenIssuer{} + +func NewTokenIssuer(signingKeyAccessor SigningKeyAccessor) TokenIssuer { + return &jwtTokenIssuer{signingKeyAccessor} +} + +type jwtTokenIssuer struct { + signingKeyAccessor SigningKeyAccessor +} + +func (j *jwtTokenIssuer) signingKey() ([]byte, error) { + signingKey, err := j.signingKeyAccessor() + if err != nil { + return nil, err + } + if len(signingKey) == 0 { + return nil, SigningKeyNotFound() + } + return signingKey, nil +} + +func (j *jwtTokenIssuer) Generate(identity Identity) (Token, error) { + signingKey, err := j.signingKey() + if err != nil { + return "", err + } + + c := claims{ + Name: identity.Name, + StandardClaims: jwt.StandardClaims{}, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, c) + tokenString, err := token.SignedString(signingKey) + if err != nil { + return "", errors.Wrap(err, "could not sign a token") + } + return tokenString, nil +} + +func (j *jwtTokenIssuer) Validate(rawToken Token) (Identity, error) { + signingKey, err := j.signingKey() + if err != nil { + return Identity{}, err + } + + c := &claims{} + + token, err := jwt.ParseWithClaims(rawToken, c, func(*jwt.Token) (interface{}, error) { + return signingKey, nil + }) + if err != nil { + return Identity{}, errors.Wrap(err, "could not parse token") + } + if !token.Valid { + return Identity{}, errors.New("token is not valid") + } + + id := Identity{ + Name: c.Name, + } + return id, nil +} diff --git a/pkg/tokens/builtin/zoneingress/signing_key.go b/pkg/tokens/builtin/zoneingress/signing_key.go new file mode 100644 index 000000000000..caefc3c514cc --- /dev/null +++ b/pkg/tokens/builtin/zoneingress/signing_key.go @@ -0,0 +1,62 @@ +package zoneingress + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + + "github.com/golang/protobuf/ptypes/wrappers" + "github.com/pkg/errors" + + system_proto "github.com/kumahq/kuma/api/system/v1alpha1" + "github.com/kumahq/kuma/pkg/core/resources/apis/system" + "github.com/kumahq/kuma/pkg/core/resources/manager" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" + "github.com/kumahq/kuma/pkg/core/resources/store" +) + +const ( + defaultRsaBits = 2048 + signingKeyName = "zone-ingress-token-signing-key" +) + +func SigningKeyNotFound() error { + return errors.Errorf("there is no Zone Ingress Signing Key in the Control Plane.") +} + +func SigningKeyResourceKey() core_model.ResourceKey { + return core_model.ResourceKey{ + Mesh: core_model.NoMesh, + Name: signingKeyName, + } +} + +func IsSigningKeyResource(resKey core_model.ResourceKey) bool { + return resKey.Name == signingKeyName && resKey.Mesh == core_model.NoMesh +} + +func GetSigningKey(manager manager.ReadOnlyResourceManager) ([]byte, error) { + resource := system.NewGlobalSecretResource() + if err := manager.Get(context.Background(), resource, store.GetBy(SigningKeyResourceKey())); err != nil { + if store.IsResourceNotFound(err) { + return nil, SigningKeyNotFound() + } + return nil, errors.Wrap(err, "could not retrieve global signing key from secret manager") + } + return resource.Spec.GetData().GetValue(), nil +} + +func CreateSigningKey() (*system.GlobalSecretResource, error) { + res := system.NewGlobalSecretResource() + key, err := rsa.GenerateKey(rand.Reader, defaultRsaBits) + if err != nil { + return res, errors.Wrap(err, "failed to generate rsa key") + } + res.Spec = &system_proto.Secret{ + Data: &wrappers.BytesValue{ + Value: x509.MarshalPKCS1PrivateKey(key), + }, + } + return res, nil +} diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index f9ea1a6ea147..05e6b225f029 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -9,6 +9,10 @@ import ( "github.com/sethvargo/go-retry" "google.golang.org/grpc/metadata" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_manager "github.com/kumahq/kuma/pkg/core/resources/manager" core_store "github.com/kumahq/kuma/pkg/core/resources/store" @@ -85,6 +89,7 @@ func (a *authCallbacks) OnStreamRequest(streamID core_xds.StreamID, req util_xds return err } err = a.authenticate(credential, req) + core.Log.WithName("TEST").Error(err, "auth.OnStreamRequest") if err != nil { return err } @@ -117,14 +122,57 @@ func (a *authCallbacks) credential(streamID core_xds.StreamID) (Credential, erro } func (a *authCallbacks) authenticate(credential Credential, req util_xds.DiscoveryRequest) error { + proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()) + if err != nil { + return errors.Wrap(err, "request must have a valid Proxy Id") + } + switch proxyId.ProxyType { + case mesh_proto.IngressDpType: + return a.authenticateZoneIngress(credential, req) + default: + return a.authenticateRegularDataplane(credential, req) + } +} + +func (a *authCallbacks) authenticateZoneIngress(credential Credential, req util_xds.DiscoveryRequest) error { + zoneIngress := core_mesh.NewZoneIngressResource() + md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) + if md.GetZoneIngressResource() != nil { + zoneIngress = md.GetZoneIngressResource() + } else { + proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()) + if err != nil { + return errors.Wrap(err, "request must have a valid Proxy Id") + } + backoff, _ := retry.NewConstant(a.dpNotFoundRetry.Backoff) + backoff = retry.WithMaxRetries(uint64(a.dpNotFoundRetry.MaxTimes), backoff) + err = retry.Do(context.Background(), backoff, func(ctx context.Context) error { + err := a.resManager.Get(ctx, zoneIngress, core_store.GetByKey(proxyId.Name, core_model.NoMesh)) + if core_store.IsResourceNotFound(err) { + return retry.RetryableError(errors.New("zoneIngress not found. Create ZoneIngress in Kuma CP first or pass it as an argument to kuma-dp")) + } + return err + }) + if err != nil { + return err + } + } + + if err := a.authenticator.AuthenticateZoneIngress(context.Background(), zoneIngress, credential); err != nil { + return errors.Wrap(err, "authentication failed") + } + return nil +} + +func (a *authCallbacks) authenticateRegularDataplane(credential Credential, req util_xds.DiscoveryRequest) error { dataplane := core_mesh.NewDataplaneResource() md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) - if md.DataplaneResource != nil { - dataplane = md.DataplaneResource + if md.GetDataplaneResource() != nil { + dataplane = md.GetDataplaneResource() } else { proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()) if err != nil { - return errors.Wrap(err, "SDS request must have a valid Proxy Id") + return errors.Wrap(err, "request must have a valid Proxy Id") } backoff, _ := retry.NewConstant(a.dpNotFoundRetry.Backoff) backoff = retry.WithMaxRetries(uint64(a.dpNotFoundRetry.MaxTimes), backoff) diff --git a/pkg/xds/auth/callbacks_test.go b/pkg/xds/auth/callbacks_test.go index 35938a74595e..31dcabc6b95f 100644 --- a/pkg/xds/auth/callbacks_test.go +++ b/pkg/xds/auth/callbacks_test.go @@ -39,6 +39,10 @@ func (t *testAuthenticator) Authenticate(ctx context.Context, dataplane *core_me return errors.New("invalid credential") } +func (t *testAuthenticator) AuthenticateZoneIngress(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, credential auth.Credential) error { + return nil +} + var _ = Describe("Auth Callbacks", func() { var testAuth *testAuthenticator diff --git a/pkg/xds/auth/components/components.go b/pkg/xds/auth/components/components.go index b14d69a625d3..b3a5ffd4000a 100644 --- a/pkg/xds/auth/components/components.go +++ b/pkg/xds/auth/components/components.go @@ -29,7 +29,11 @@ func NewUniversalAuthenticator(rt core_runtime.Runtime) (auth.Authenticator, err if err != nil { return nil, err } - return universal_auth.NewAuthenticator(issuer), nil + zoneIngressIssuer, err := builtin.NewZoneIngressTokenIssuer(rt.ReadOnlyResourceManager()) + if err != nil { + return nil, err + } + return universal_auth.NewAuthenticator(issuer, zoneIngressIssuer), nil } func DefaultAuthenticator(rt core_runtime.Runtime) (auth.Authenticator, error) { diff --git a/pkg/xds/auth/interfaces.go b/pkg/xds/auth/interfaces.go index 6fdf50ee4c6a..c5f3649942ae 100644 --- a/pkg/xds/auth/interfaces.go +++ b/pkg/xds/auth/interfaces.go @@ -10,4 +10,5 @@ type Credential = string type Authenticator interface { Authenticate(ctx context.Context, dataplane *core_mesh.DataplaneResource, credential Credential) error + AuthenticateZoneIngress(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, credential Credential) error } diff --git a/pkg/xds/auth/k8s/authenticator.go b/pkg/xds/auth/k8s/authenticator.go index 179592068dab..16a4ee12f8c4 100644 --- a/pkg/xds/auth/k8s/authenticator.go +++ b/pkg/xds/auth/k8s/authenticator.go @@ -56,3 +56,36 @@ func (k *kubeAuthenticator) Authenticate(ctx context.Context, dataplane *core_me } return nil } + +func (k *kubeAuthenticator) AuthenticateZoneIngress(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, credential auth.Credential) error { + if credential == "" { + return errors.New("authentication failed: k8s token is missing") + } + tokenReview := &kube_auth.TokenReview{ + Spec: kube_auth.TokenReviewSpec{ + Token: credential, + }, + } + if err := k.client.Create(ctx, tokenReview); err != nil { + return errors.Wrap(err, "authentication failed: call to TokenReview API failed") + } + if !tokenReview.Status.Authenticated { + return errors.Errorf("authentication failed: token doesn't belong to a valid user") + } + userInfo := strings.Split(tokenReview.Status.User.Username, ":") + if len(userInfo) != 4 { + return errors.Errorf("authentication failed: username inside TokenReview response has unexpected format: %q", tokenReview.Status.User.Username) + } + if !(userInfo[0] == "system" && userInfo[1] == "serviceaccount") { + return errors.Errorf("authentication failed: token must belong to a k8s system account, got %q", tokenReview.Status.User.Username) + } + _, proxyNamespace, err := util_k8s.CoreNameToK8sName(zoneIngress.Meta.GetName()) + if err != nil { + return err + } + namespace := userInfo[2] + if namespace != proxyNamespace { + return errors.Errorf("authentication failed: token belongs to a namespace (%q) different from proxyId (%q)", namespace, proxyNamespace) + } + return nil +} diff --git a/pkg/xds/auth/universal/auth_test.go b/pkg/xds/auth/universal/auth_test.go index 41cd7d8dff58..403bd1440b2f 100644 --- a/pkg/xds/auth/universal/auth_test.go +++ b/pkg/xds/auth/universal/auth_test.go @@ -7,6 +7,8 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_store "github.com/kumahq/kuma/pkg/core/resources/store" @@ -23,6 +25,9 @@ var _ = Describe("Authentication flow", func() { issuer := builtin_issuer.NewDataplaneTokenIssuer(func(string) ([]byte, error) { return privateKey, nil }) + zoneIngressIssuer := zoneingress.NewTokenIssuer(func() ([]byte, error) { + return privateKey, nil + }) var authenticator auth.Authenticator var resStore core_store.ResourceStore @@ -79,7 +84,7 @@ var _ = Describe("Authentication flow", func() { BeforeEach(func() { resStore = memory.NewStore() - authenticator = universal.NewAuthenticator(issuer) + authenticator = universal.NewAuthenticator(issuer, zoneIngressIssuer) err := resStore.Create(context.Background(), &dpRes, core_store.CreateByKey("dp-1", "default")) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/xds/auth/universal/authenticator.go b/pkg/xds/auth/universal/authenticator.go index a9a00a913d9a..22aef9b28be6 100644 --- a/pkg/xds/auth/universal/authenticator.go +++ b/pkg/xds/auth/universal/authenticator.go @@ -5,15 +5,18 @@ import ( "github.com/pkg/errors" + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" builtin_issuer "github.com/kumahq/kuma/pkg/tokens/builtin/issuer" "github.com/kumahq/kuma/pkg/xds/auth" ) -func NewAuthenticator(issuer builtin_issuer.DataplaneTokenIssuer) auth.Authenticator { +func NewAuthenticator(issuer builtin_issuer.DataplaneTokenIssuer, zoneIngressIssuer zoneingress.TokenIssuer) auth.Authenticator { return &universalAuthenticator{ - issuer: issuer, + issuer: issuer, + zoneIngressIssuer: zoneIngressIssuer, } } @@ -27,7 +30,8 @@ func NewAuthenticator(issuer builtin_issuer.DataplaneTokenIssuer) auth.Authentic // with inbounds: 1) kuma.io/service:web 2) kuma.io/service:web-api, you need token for both values kuma.io/service=web,web-api // Dataplane also needs to have all tags defined in the token type universalAuthenticator struct { - issuer builtin_issuer.DataplaneTokenIssuer + issuer builtin_issuer.DataplaneTokenIssuer + zoneIngressIssuer zoneingress.TokenIssuer } func (u *universalAuthenticator) Authenticate(ctx context.Context, dataplane *core_mesh.DataplaneResource, credential auth.Credential) error { @@ -51,6 +55,18 @@ func (u *universalAuthenticator) Authenticate(ctx context.Context, dataplane *co return nil } +func (u *universalAuthenticator) AuthenticateZoneIngress(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, credential auth.Credential) error { + identity, err := u.zoneIngressIssuer.Validate(credential) + if err != nil { + return err + } + if zoneIngress.Meta.GetName() != identity.Name { + return errors.Errorf("zone ingress name from requestor: %s is different than in token: %s", zoneIngress.Meta.GetName(), identity.Name) + } + + return nil +} + func validateType(dataplane *core_mesh.DataplaneResource, dpType builtin_issuer.DpType) error { if dpType == "" { // if dp type is not explicitly specified we assume it's dataplane so we force Ingress token dpType = builtin_issuer.DpTypeDataplane diff --git a/pkg/xds/auth/universal/noop_authenticator.go b/pkg/xds/auth/universal/noop_authenticator.go index 31d4e77da1ff..59ac21ee3b0d 100644 --- a/pkg/xds/auth/universal/noop_authenticator.go +++ b/pkg/xds/auth/universal/noop_authenticator.go @@ -17,3 +17,7 @@ type noopAuthenticator struct { func (u *noopAuthenticator) Authenticate(ctx context.Context, dataplane *core_mesh.DataplaneResource, _ auth.Credential) error { return nil } + +func (u *noopAuthenticator) AuthenticateZoneIngress(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, _ auth.Credential) error { + return nil +} diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 8a3541eac33c..76dd47be9354 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -16,7 +16,10 @@ import ( "github.com/asaskevich/govalidator" - "github.com/kumahq/kuma/pkg/core/resources/model" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" + + core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/resources/model/rest" "github.com/kumahq/kuma/pkg/core/validators" envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" @@ -78,17 +81,34 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr return nil, "", err } - proxyId, err := core_xds.BuildProxyId(request.Mesh, request.Name) - if err != nil { - return nil, "", err - } - - dataplane, err := b.dataplaneFor(ctx, request, proxyId) - if err != nil { - return nil, "", err + switch mesh_proto.DpType(request.ProxyType) { + case mesh_proto.IngressDpType: + proxyId := core_xds.BuildProxyId(request.Mesh, request.Name, mesh_proto.IngressDpType) + zoneIngress, err := b.zoneIngressFor(ctx, request, proxyId) + if err != nil { + return nil, "", err + } + adminPort, err := b.adminPortForIngress(request, zoneIngress) + if err != nil { + return nil, "", err + } + b.hdsEnabled = false + return b.generateFor(*proxyId, request, "ingress", adminPort) + case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: + proxyId := core_xds.BuildProxyId(request.Mesh, request.Name, mesh_proto.RegularDpType) + dataplane, err := b.dataplaneFor(ctx, request, proxyId) + if err != nil { + return nil, "", err + } + service := dataplane.Spec.GetIdentifyingService() + adminPort, err := b.adminPortForDataplane(request, dataplane) + if err != nil { + return nil, "", err + } + return b.generateFor(*proxyId, request, service, adminPort) + default: + return nil, "", nil } - - return b.generateFor(*proxyId, dataplane, request) } var DpTokenRequired = errors.New("Dataplane Token is required. Generate token using 'kumactl generate dataplane-token > /path/file' and provide it via --dataplane-token-file=/path/file argument to Kuma DP") @@ -170,8 +190,33 @@ func (b *bootstrapGenerator) dataplaneFor(ctx context.Context, request types.Boo } } +func (b *bootstrapGenerator) zoneIngressFor(ctx context.Context, request types.BootstrapRequest, proxyId *core_xds.ProxyId) (*core_mesh.ZoneIngressResource, error) { + if request.DataplaneResource != "" { + res, err := rest.UnmarshallToCore([]byte(request.DataplaneResource)) + if err != nil { + return nil, err + } + zoneIngress, ok := res.(*core_mesh.ZoneIngressResource) + if !ok { + return nil, errors.Errorf("invalid resource") + } + if err := zoneIngress.Validate(); err != nil { + return nil, err + } + return zoneIngress, nil + } else { + zoneIngress := core_mesh.NewZoneIngressResource() + core.Log.WithName("TEST").Info("zoneIngressFor", "key", proxyId.Name) + if err := b.resManager.Get(ctx, zoneIngress, core_store.GetByKey(proxyId.Name, core_model.NoMesh)); err != nil { + core.Log.WithName("TEST").Error(err, "zoneIngressFor") + return nil, err + } + return zoneIngress, nil + } +} + func (b *bootstrapGenerator) validateMeshExist(ctx context.Context, mesh string) error { - if err := b.resManager.Get(ctx, core_mesh.NewMeshResource(), core_store.GetByKey(mesh, model.NoMesh)); err != nil { + if err := b.resManager.Get(ctx, core_mesh.NewMeshResource(), core_store.GetByKey(mesh, core_model.NoMesh)); err != nil { if core_store.IsResourceNotFound(err) { verr := validators.ValidationError{} verr.AddViolation("mesh", fmt.Sprintf("mesh %q does not exist", mesh)) @@ -182,19 +227,34 @@ func (b *bootstrapGenerator) validateMeshExist(ctx context.Context, mesh string) return nil } -func (b *bootstrapGenerator) generateFor(proxyId core_xds.ProxyId, dataplane *core_mesh.DataplaneResource, request types.BootstrapRequest) (proto.Message, types.BootstrapVersion, error) { - // if dataplane has no service - fill this with placeholder. Otherwise take the first service - service := dataplane.Spec.GetIdentifyingService() - +func (b *bootstrapGenerator) adminPortForDataplane(request types.BootstrapRequest, dataplane *core_mesh.DataplaneResource) (uint32, error) { adminPort := b.config.Params.AdminPort if request.AdminPort != 0 { adminPort = request.AdminPort } + // The admin port in kuma-dp is always bound to 127.0.0.1 + if dataplane.UsesInboundInterface(core_mesh.IPv4Loopback, adminPort) { + return 0, errors.Errorf("Resource precondition failed: Port %d requested as both admin and inbound port.", adminPort) + } + if dataplane.UsesOutboundInterface(core_mesh.IPv4Loopback, adminPort) { + return 0, errors.Errorf("Resource precondition failed: Port %d requested as both admin and outbound port.", adminPort) + } + return adminPort, nil +} - if err := b.verifyAdminPort(adminPort, dataplane); err != nil { - return nil, "", err +func (b *bootstrapGenerator) adminPortForIngress(request types.BootstrapRequest, zoneIngress *core_mesh.ZoneIngressResource) (uint32, error) { + adminPort := b.config.Params.AdminPort + if request.AdminPort != 0 { + adminPort = request.AdminPort + } + // The admin port in kuma-dp is always bound to 127.0.0.1 + if zoneIngress.UsesInboundInterface(core_mesh.IPv4Loopback, adminPort) { + return 0, errors.Errorf("Resource precondition failed: Port %d requested as both admin and inbound port.", adminPort) } + return adminPort, nil +} +func (b *bootstrapGenerator) generateFor(proxyId core_xds.ProxyId, request types.BootstrapRequest, service string, adminPort uint32) (proto.Message, types.BootstrapVersion, error) { cert, origin, err := b.caCert(request) if err != nil { return nil, "", err @@ -287,18 +347,6 @@ func (b *bootstrapGenerator) xdsHost(request types.BootstrapRequest) string { } } -func (b *bootstrapGenerator) verifyAdminPort(adminPort uint32, dataplane *core_mesh.DataplaneResource) error { - // The admin port in kuma-dp is always bound to 127.0.0.1 - if dataplane.UsesInboundInterface(core_mesh.IPv4Loopback, adminPort) { - return errors.Errorf("Resource precondition failed: Port %d requested as both admin and inbound port.", adminPort) - } - - if dataplane.UsesOutboundInterface(core_mesh.IPv4Loopback, adminPort) { - return errors.Errorf("Resource precondition failed: Port %d requested as both admin and outbound port.", adminPort) - } - return nil -} - func (b *bootstrapGenerator) bootstrapVersion(reqVersion types.BootstrapVersion) types.BootstrapVersion { if reqVersion != "" { return reqVersion diff --git a/pkg/xds/bootstrap/types/bootstrap_request.go b/pkg/xds/bootstrap/types/bootstrap_request.go index 37cc88a9202d..6ee4f3cd627c 100644 --- a/pkg/xds/bootstrap/types/bootstrap_request.go +++ b/pkg/xds/bootstrap/types/bootstrap_request.go @@ -15,6 +15,7 @@ const BootstrapVersionHeader = "kuma-bootstrap-version" type BootstrapRequest struct { Mesh string `json:"mesh"` Name string `json:"name"` + ProxyType string `json:"proxyType"` AdminPort uint32 `json:"adminPort,omitempty"` DataplaneTokenPath string `json:"dataplaneTokenPath,omitempty"` DataplaneToken string `json:"dataplaneToken,omitempty"` diff --git a/pkg/xds/cache/cla/cache.go b/pkg/xds/cache/cla/cache.go index f00c23bd54fe..9354ef611a81 100644 --- a/pkg/xds/cache/cla/cache.go +++ b/pkg/xds/cache/cla/cache.go @@ -62,6 +62,10 @@ func (c *Cache) GetCLA(ctx context.Context, meshName, meshHash string, cluster e if err != nil { return nil, err } + zoneIngresses := &core_mesh.ZoneIngressResourceList{} + if err := c.rm.List(ctx, zoneIngresses); err != nil { + return nil, err + } mesh := core_mesh.NewMeshResource() if err := c.rm.Get(ctx, mesh, core_store.GetByKey(meshName, model.NoMesh)); err != nil { return nil, err @@ -71,7 +75,7 @@ func (c *Cache) GetCLA(ctx context.Context, meshName, meshHash string, cluster e // This also solves the problem that if the ExternalService is blocked by TrafficPermission // OutboundProxyGenerate treats this as EDS cluster and tries to get endpoints via GetCLA // Since GetCLA is consistent for a mesh, it would return an endpoint with address which is not valid for EDS. - endpointMap := topology.BuildEdsEndpointMap(mesh, c.zone, dataplanes.Items) + endpointMap := topology.BuildEdsEndpointMap(mesh, c.zone, dataplanes.Items, zoneIngresses.Items) endpoints := []xds.Endpoint{} for _, endpoint := range endpointMap[cluster.Service()] { if endpoint.ContainsTags(cluster.Tags()) { diff --git a/pkg/xds/cache/mesh/snapshot.go b/pkg/xds/cache/mesh/snapshot.go index af2444621819..e5a95b5c2004 100644 --- a/pkg/xds/cache/mesh/snapshot.go +++ b/pkg/xds/cache/mesh/snapshot.go @@ -47,6 +47,7 @@ func GetMeshSnapshot(ctx context.Context, meshName string, rm manager.ReadOnlyRe if err := rm.List(ctx, dataplanes); err != nil { return nil, err } + // backwards compatibility meshedDpsAndIngresses := &core_mesh.DataplaneResourceList{} for _, d := range dataplanes.Items { if d.GetMeta().GetMesh() == meshName || d.Spec.IsIngress() { @@ -54,6 +55,12 @@ func GetMeshSnapshot(ctx context.Context, meshName string, rm manager.ReadOnlyRe } } snapshot.resources[typ] = meshedDpsAndIngresses + case core_mesh.ZoneIngressType: + zoneIngresses := &core_mesh.ZoneIngressResourceList{} + if err := rm.List(ctx, zoneIngresses); err != nil { + return nil, err + } + snapshot.resources[typ] = zoneIngresses case system.ConfigType: configs := &system.ConfigResourceList{} var items []*system.ConfigResource diff --git a/pkg/xds/generator/admin_proxy_generator.go b/pkg/xds/generator/admin_proxy_generator.go index 14265e9f031a..9b688570950c 100644 --- a/pkg/xds/generator/admin_proxy_generator.go +++ b/pkg/xds/generator/admin_proxy_generator.go @@ -16,14 +16,14 @@ import ( // OriginAdmin is a marker to indicate by which ProxyGenerator resources were generated. const OriginAdmin = "admin" -var staticEnpointPaths = []*envoy_common.StaticEndpointPath{ +var staticEndpointPaths = []*envoy_common.StaticEndpointPath{ { Path: "/ready", RewritePath: "/ready", }, } -var staticTlsEnpointPaths = []*envoy_common.StaticEndpointPath{ +var staticTlsEndpointPaths = []*envoy_common.StaticEndpointPath{ { Path: "/", RewritePath: "/", @@ -59,34 +59,37 @@ func (g AdminProxyGenerator) Generate(ctx xds_context.Context, proxy *core_xds.P resources := core_xds.NewResourceSet() - for _, se := range staticEnpointPaths { + for _, se := range staticEndpointPaths { se.ClusterName = envoyAdminClusterName } - for _, se := range staticTlsEnpointPaths { - se.ClusterName = envoyAdminClusterName - - token, err := ctx.EnvoyAdminClient.GenerateAPIToken(proxy.Dataplane) - if err != nil { - return nil, errors.Wrapf(err, "unable to generate the API token") + // We bind admin to 127.0.0.1 by default, creating another listener with same address and port will result in error. + if g.getAddress(proxy) != "127.0.0.1" { + filterChains := []envoy_listeners.ListenerBuilderOpt{ + envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion). + Configure(envoy_listeners.StaticEndpoints(envoy_names.GetAdminListenerName(), staticEndpointPaths)), + ), } - se.Header = "Authorization" - se.HeaderExactMatch = fmt.Sprintf("Bearer %s", token) - } + if proxy.Dataplane != nil { + for _, se := range staticTlsEndpointPaths { + se.ClusterName = envoyAdminClusterName - // We bind admin to 127.0.0.1 by default, creating another listener with same address and port will result in error. - if proxy.Dataplane.Spec.GetNetworking().Address != "127.0.0.1" { + token, err := ctx.EnvoyAdminClient.GenerateAPIToken(proxy.Dataplane) + if err != nil { + return nil, errors.Wrapf(err, "unable to generate the API token") + } + se.Header = "Authorization" + se.HeaderExactMatch = fmt.Sprintf("Bearer %s", token) + } + filterChains = append(filterChains, envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion). + Configure(envoy_listeners.FilterChainMatch("tls")). + Configure(envoy_listeners.StaticTlsEndpoints(envoy_names.GetAdminListenerName(), ctx.ControlPlane.AdminProxyKeyPair, staticTlsEndpointPaths)), + )) + } listener, err := envoy_listeners.NewListenerBuilder(proxy.APIVersion). - Configure(envoy_listeners.InboundListener(envoy_names.GetAdminListenerName(), proxy.Dataplane.Spec.GetNetworking().Address, adminPort, core_xds.SocketAddressProtocolTCP)). + Configure(envoy_listeners.InboundListener(envoy_names.GetAdminListenerName(), g.getAddress(proxy), adminPort, core_xds.SocketAddressProtocolTCP)). Configure(envoy_listeners.TLSInspector()). - Configure( - envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion). - Configure(envoy_listeners.StaticEndpoints(envoy_names.GetAdminListenerName(), staticEnpointPaths)), - ), - envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion). - Configure(envoy_listeners.FilterChainMatch("tls")). - Configure(envoy_listeners.StaticTlsEndpoints(envoy_names.GetAdminListenerName(), ctx.ControlPlane.AdminProxyKeyPair, staticTlsEnpointPaths)), - )). + Configure(filterChains...). Build() if err != nil { return nil, err @@ -105,3 +108,10 @@ func (g AdminProxyGenerator) Generate(ctx xds_context.Context, proxy *core_xds.P }) return resources, nil } + +func (g AdminProxyGenerator) getAddress(proxy *core_xds.Proxy) string { + if proxy.Dataplane != nil { + return proxy.Dataplane.Spec.GetNetworking().Address + } + return proxy.ZoneIngress.Spec.Address +} diff --git a/pkg/xds/generator/ingress_generator.go b/pkg/xds/generator/ingress_generator.go index 2b823befa583..696fc074f5ac 100644 --- a/pkg/xds/generator/ingress_generator.go +++ b/pkg/xds/generator/ingress_generator.go @@ -3,8 +3,6 @@ package generator import ( "sort" - "github.com/kumahq/kuma/pkg/xds/envoy/tls" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" model "github.com/kumahq/kuma/pkg/core/xds" @@ -14,6 +12,7 @@ import ( envoy_endpoints "github.com/kumahq/kuma/pkg/xds/envoy/endpoints" envoy_listeners "github.com/kumahq/kuma/pkg/xds/envoy/listeners" envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names" + "github.com/kumahq/kuma/pkg/xds/envoy/tls" ) const ( @@ -31,7 +30,7 @@ func (i IngressGenerator) Generate(ctx xds_context.Context, proxy *model.Proxy) destinationsPerService := i.destinations(proxy.Routing.TrafficRouteList) - listener, err := i.generateLDS(proxy.Dataplane, destinationsPerService, proxy.APIVersion) + listener, err := i.generateLDS(proxy, proxy.ZoneIngress, destinationsPerService, proxy.APIVersion) if err != nil { return nil, err } @@ -65,24 +64,24 @@ func (i IngressGenerator) Generate(ctx xds_context.Context, proxy *model.Proxy) // This approach has a limitation: additional tags on outbound in Universal mode won't work across different zones. // Traffic is NOT decrypted here, therefore we don't need certificates and mTLS settings func (i IngressGenerator) generateLDS( - ingress *core_mesh.DataplaneResource, + proxy *model.Proxy, + ingress *core_mesh.ZoneIngressResource, destinationsPerService map[string][]envoy_common.Tags, apiVersion envoy_common.APIVersion, ) (envoy_common.NamedResource, error) { - inbound := ingress.Spec.Networking.Inbound[0] - inboundListenerName := envoy_names.GetInboundListenerName(ingress.Spec.GetNetworking().GetAddress(), inbound.Port) + inboundListenerName := envoy_names.GetInboundListenerName(proxy.ZoneIngress.Spec.GetAddress(), proxy.ZoneIngress.Spec.GetPort()) inboundListenerBuilder := envoy_listeners.NewListenerBuilder(apiVersion). - Configure(envoy_listeners.InboundListener(inboundListenerName, ingress.Spec.GetNetworking().GetAddress(), inbound.Port, model.SocketAddressProtocolTCP)). + Configure(envoy_listeners.InboundListener(inboundListenerName, ingress.Spec.GetAddress(), ingress.Spec.GetPort(), model.SocketAddressProtocolTCP)). Configure(envoy_listeners.TLSInspector()) - if !ingress.Spec.HasAvailableServices() { + if len(proxy.ZoneIngress.Spec.AvailableServices) == 0 { inboundListenerBuilder = inboundListenerBuilder. Configure(envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(apiVersion))) } sniUsed := map[string]bool{} - for _, inbound := range ingress.Spec.GetNetworking().GetIngress().GetAvailableServices() { + for _, inbound := range proxy.ZoneIngress.Spec.GetAvailableServices() { service := inbound.Tags[mesh_proto.ServiceTag] destinations := destinationsPerService[service] destinations = append(destinations, destinationsPerService[mesh_proto.MatchAllTag]...) diff --git a/pkg/xds/generator/ingress_generator_test.go b/pkg/xds/generator/ingress_generator_test.go index a2c317cec77f..94abbe17b961 100644 --- a/pkg/xds/generator/ingress_generator_test.go +++ b/pkg/xds/generator/ingress_generator_test.go @@ -7,6 +7,8 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_xds "github.com/kumahq/kuma/pkg/core/xds" @@ -14,7 +16,6 @@ import ( test_model "github.com/kumahq/kuma/pkg/test/resources/model" util_proto "github.com/kumahq/kuma/pkg/util/proto" xds_context "github.com/kumahq/kuma/pkg/xds/context" - envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" "github.com/kumahq/kuma/pkg/xds/generator" ) @@ -33,15 +34,17 @@ var _ = Describe("IngressGenerator", func() { dataplane := &mesh_proto.Dataplane{} Expect(util_proto.FromYAML([]byte(given.dataplane), dataplane)).To(Succeed()) - proxy := &core_xds.Proxy{ - Id: core_xds.ProxyId{Name: "ingress", Mesh: "default"}, - Dataplane: &mesh_core.DataplaneResource{ - Meta: &test_model.ResourceMeta{ - Version: "1", - }, - Spec: dataplane, + zoneIngress, err := mesh_core.NewZoneIngressResourceFromDataplane(&mesh_core.DataplaneResource{ + Meta: &test_model.ResourceMeta{ + Version: "1", }, - APIVersion: envoy_common.APIV3, + Spec: dataplane, + }) + Expect(err).ToNot(HaveOccurred()) + proxy := &core_xds.Proxy{ + Id: core_xds.ProxyId{Name: "ingress", Mesh: "default"}, + ZoneIngress: zoneIngress, + APIVersion: envoy_common.APIV3, Routing: core_xds.Routing{ OutboundTargets: given.outboundTargets, TrafficRouteList: given.trafficRoutes, diff --git a/pkg/xds/ingress/dataplane.go b/pkg/xds/ingress/dataplane.go index a4615245d406..3b902d9c274d 100644 --- a/pkg/xds/ingress/dataplane.go +++ b/pkg/xds/ingress/dataplane.go @@ -38,7 +38,7 @@ func (s tagSets) addInstanceOfTags(mesh string, tags envoy.Tags) { s[serviceKey{tags: strTags, mesh: mesh}]++ } -func (s tagSets) toAvailableServices() []*mesh_proto.Dataplane_Networking_Ingress_AvailableService { +func (s tagSets) toAvailableServicesCompat() []*mesh_proto.Dataplane_Networking_Ingress_AvailableService { var result []*mesh_proto.Dataplane_Networking_Ingress_AvailableService var keys []serviceKey @@ -58,9 +58,29 @@ func (s tagSets) toAvailableServices() []*mesh_proto.Dataplane_Networking_Ingres return result } -func UpdateAvailableServices(ctx context.Context, rm manager.ResourceManager, ingress *core_mesh.DataplaneResource, others []*core_mesh.DataplaneResource) error { - availableServices := GetIngressAvailableServices(others) - if availableServicesEqual(availableServices, ingress.Spec.GetNetworking().GetIngress().GetAvailableServices()) { +func (s tagSets) toAvailableServices() []*mesh_proto.ZoneIngress_AvailableService { + var result []*mesh_proto.ZoneIngress_AvailableService + + var keys []serviceKey + for key := range s { + keys = append(keys, key) + } + sort.Sort(serviceKeySlice(keys)) + + for _, key := range keys { + tags, _ := envoy.TagsFromString(key.tags) // ignore error since we control how string looks like + result = append(result, &mesh_proto.ZoneIngress_AvailableService{ + Tags: tags, + Instances: s[key], + Mesh: key.mesh, + }) + } + return result +} + +func UpdateAvailableServicesCompat(ctx context.Context, rm manager.ResourceManager, ingress *core_mesh.DataplaneResource, others []*core_mesh.DataplaneResource) error { + availableServices := GetIngressAvailableServicesCompat(others) + if availableServicesEqualCompat(availableServices, ingress.Spec.GetNetworking().GetIngress().GetAvailableServices()) { return nil } ingress.Spec.Networking.Ingress.AvailableServices = availableServices @@ -70,7 +90,31 @@ func UpdateAvailableServices(ctx context.Context, rm manager.ResourceManager, in return nil } -func availableServicesEqual(services []*mesh_proto.Dataplane_Networking_Ingress_AvailableService, other []*mesh_proto.Dataplane_Networking_Ingress_AvailableService) bool { +func UpdateAvailableServices(ctx context.Context, rm manager.ResourceManager, ingress *core_mesh.ZoneIngressResource, others []*core_mesh.DataplaneResource) error { + availableServices := GetIngressAvailableServices(others) + if availableServicesEqual(availableServices, ingress.Spec.GetAvailableServices()) { + return nil + } + ingress.Spec.AvailableServices = availableServices + if err := rm.Update(ctx, ingress); err != nil { + return err + } + return nil +} + +func availableServicesEqualCompat(services []*mesh_proto.Dataplane_Networking_Ingress_AvailableService, other []*mesh_proto.Dataplane_Networking_Ingress_AvailableService) bool { + if len(services) != len(other) { + return false + } + for i := range services { + if !proto.Equal(services[i], other[i]) { + return false + } + } + return true +} + +func availableServicesEqual(services []*mesh_proto.ZoneIngress_AvailableService, other []*mesh_proto.ZoneIngress_AvailableService) bool { if len(services) != len(other) { return false } @@ -82,7 +126,20 @@ func availableServicesEqual(services []*mesh_proto.Dataplane_Networking_Ingress_ return true } -func GetIngressAvailableServices(others []*core_mesh.DataplaneResource) []*mesh_proto.Dataplane_Networking_Ingress_AvailableService { +func GetIngressAvailableServicesCompat(others []*core_mesh.DataplaneResource) []*mesh_proto.Dataplane_Networking_Ingress_AvailableService { + tagSets := tagSets{} + for _, dp := range others { + if dp.Spec.IsIngress() { + continue + } + for _, dpInbound := range dp.Spec.GetNetworking().GetHealthyInbounds() { + tagSets.addInstanceOfTags(dp.GetMeta().GetMesh(), dpInbound.Tags) + } + } + return tagSets.toAvailableServicesCompat() +} + +func GetIngressAvailableServices(others []*core_mesh.DataplaneResource) []*mesh_proto.ZoneIngress_AvailableService { tagSets := tagSets{} for _, dp := range others { if dp.Spec.IsIngress() { diff --git a/pkg/xds/ingress/dataplane_test.go b/pkg/xds/ingress/dataplane_test.go index 4aa8a76d6152..865921b6f2d3 100644 --- a/pkg/xds/ingress/dataplane_test.go +++ b/pkg/xds/ingress/dataplane_test.go @@ -164,9 +164,12 @@ var _ = Describe("Ingress Dataplane", func() { ctx := context.Background() mgr := &fakeResourceManager{} - ing := &core_mesh.DataplaneResource{ + ing, err := core_mesh.NewZoneIngressResourceFromDataplane(&core_mesh.DataplaneResource{ Spec: &mesh_proto.Dataplane{ Networking: &mesh_proto.Dataplane_Networking{ + Inbound: []*mesh_proto.Dataplane_Networking_Inbound{{ + Port: 10001, + }}, Ingress: &mesh_proto.Dataplane_Networking_Ingress{ AvailableServices: []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{ { @@ -191,7 +194,8 @@ var _ = Describe("Ingress Dataplane", func() { }, }, }, - } + }) + Expect(err).ToNot(HaveOccurred()) others := []*core_mesh.DataplaneResource{ { @@ -243,7 +247,7 @@ var _ = Describe("Ingress Dataplane", func() { }, }, } - err := ingress.UpdateAvailableServices(ctx, mgr, ing, others) + err = ingress.UpdateAvailableServices(ctx, mgr, ing, others) Expect(err).ToNot(HaveOccurred()) Expect(mgr.updCounter).To(Equal(0)) }) @@ -299,7 +303,7 @@ var _ = Describe("Ingress Dataplane", func() { }, }, } - expectedAvailableServices := []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{ + expectedAvailableServices := []*mesh_proto.ZoneIngress_AvailableService{ { Instances: 1, Tags: map[string]string{ @@ -368,7 +372,7 @@ var _ = Describe("Ingress Dataplane", func() { }, }, } - expectedAvailableServices := []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{ + expectedAvailableServices := []*mesh_proto.ZoneIngress_AvailableService{ { Instances: 1, Tags: map[string]string{ diff --git a/pkg/xds/ingress/router.go b/pkg/xds/ingress/router.go index d398acf6dd64..6df65a6c65fb 100644 --- a/pkg/xds/ingress/router.go +++ b/pkg/xds/ingress/router.go @@ -6,9 +6,9 @@ import ( core_xds "github.com/kumahq/kuma/pkg/core/xds" ) -func BuildDestinationMap(ingress *core_mesh.DataplaneResource) core_xds.DestinationMap { +func BuildDestinationMap(ingress *core_mesh.ZoneIngressResource) core_xds.DestinationMap { destinations := core_xds.DestinationMap{} - for _, ingress := range ingress.Spec.GetNetworking().GetIngress().GetAvailableServices() { + for _, ingress := range ingress.Spec.GetAvailableServices() { service := ingress.Tags[mesh_proto.ServiceTag] destinations[service] = destinations[service].Add(mesh_proto.MatchTags(ingress.Tags)) } diff --git a/pkg/xds/ingress/router_test.go b/pkg/xds/ingress/router_test.go index 4926d04e1570..a594a6c02ce6 100644 --- a/pkg/xds/ingress/router_test.go +++ b/pkg/xds/ingress/router_test.go @@ -2,7 +2,7 @@ package ingress import ( . "github.com/onsi/ginkgo" - "github.com/onsi/gomega" + . "github.com/onsi/gomega" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" @@ -11,9 +11,12 @@ import ( var _ = Describe("Ingress BuildDestinationMap", func() { It("should generate destination map by ingress", func() { - ingress := &mesh.DataplaneResource{ + ingress, err := mesh.NewZoneIngressResourceFromDataplane(&mesh.DataplaneResource{ Spec: &mesh_proto.Dataplane{ Networking: &mesh_proto.Dataplane_Networking{ + Inbound: []*mesh_proto.Dataplane_Networking_Inbound{{ + Port: 10001, + }}, Ingress: &mesh_proto.Dataplane_Networking_Ingress{ AvailableServices: []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{ { @@ -29,7 +32,9 @@ var _ = Describe("Ingress BuildDestinationMap", func() { }, }, }, - } + }) + Expect(err).ToNot(HaveOccurred()) + actual := BuildDestinationMap(ingress) expected := xds.DestinationMap{ "backend": []mesh_proto.TagSelector{ @@ -50,6 +55,6 @@ var _ = Describe("Ingress BuildDestinationMap", func() { }, }, } - gomega.Expect(actual).To(gomega.Equal(expected)) + Expect(actual).To(Equal(expected)) }) }) diff --git a/pkg/xds/server/callbacks/dataplane_lifecycle.go b/pkg/xds/server/callbacks/dataplane_lifecycle.go index 88d8295283b3..1963c5f8dbec 100644 --- a/pkg/xds/server/callbacks/dataplane_lifecycle.go +++ b/pkg/xds/server/callbacks/dataplane_lifecycle.go @@ -6,6 +6,8 @@ import ( "github.com/pkg/errors" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" @@ -32,6 +34,7 @@ type DataplaneLifecycle struct { // createdDpForStream stores map from StreamID to created ResourceKey of Dataplane. // we store nil values for streams without Dataplane in metadata to avoid accessing metadata with every DiscoveryRequest createdDpForStream map[xds.StreamID]*model.ResourceKey + proxyTypeForStream map[xds.StreamID]mesh_proto.DpType sync.RWMutex // protects createdDpForStream shutdownCh <-chan struct{} } @@ -42,6 +45,7 @@ func NewDataplaneLifecycle(resManager manager.ResourceManager, shutdownCh <-chan return &DataplaneLifecycle{ resManager: resManager, createdDpForStream: map[xds.StreamID]*model.ResourceKey{}, + proxyTypeForStream: map[xds.StreamID]mesh_proto.DpType{}, shutdownCh: shutdownCh, } } @@ -61,12 +65,28 @@ func (d *DataplaneLifecycle) OnStreamClosed(streamID int64) { d.Lock() defer d.Unlock() key := d.createdDpForStream[streamID] + proxyType := d.proxyTypeForStream[streamID] delete(d.createdDpForStream, streamID) - if key != nil { + delete(d.proxyTypeForStream, streamID) + + if key == nil { + return + } + + if proxyType == mesh_proto.RegularDpType { lifecycleLog.Info("unregistering dataplane", "dataplaneKey", key, "streamID", streamID) if err := d.unregisterDataplane(*key); err != nil { lifecycleLog.Error(err, "could not unregister dataplane") } + return + } + + if proxyType == mesh_proto.IngressDpType { + lifecycleLog.Info("unregistering zone ingress", "zoneIngressKey", key, "streamID", streamID) + if err := d.unregisterZoneIngress(*key); err != nil { + lifecycleLog.Error(err, "could not unregister zone ingress") + } + return } } @@ -81,17 +101,36 @@ func (d *DataplaneLifecycle) OnStreamRequest(streamID int64, request util_xds.Di d.Lock() defer d.Unlock() + md := xds.DataplaneMetadataFromXdsMetadata(request.Metadata()) - if md.DataplaneResource != nil { - lifecycleLog.Info("registering dataplane", "dataplane", md.DataplaneResource, "streamID", streamID, "nodeID", request.NodeId()) - if err := d.registerDataplane(md.DataplaneResource); err != nil { + proxyId, err := xds.ParseProxyIdFromString(request.NodeId()) + if err != nil { + return nil + } + + if proxyId.ProxyType == mesh_proto.RegularDpType && md.GetDataplaneResource() != nil { + lifecycleLog.Info("registering dataplane", "dataplane", md.GetDataplaneResource(), "streamID", streamID, "nodeID", request.NodeId()) + if err := d.registerDataplane(md.GetDataplaneResource()); err != nil { return errors.Wrap(err, "could not register dataplane passed in kuma-dp run") } - key := model.MetaToResourceKey(md.DataplaneResource.GetMeta()) + key := model.MetaToResourceKey(md.GetDataplaneResource().GetMeta()) d.createdDpForStream[streamID] = &key - } else { - d.createdDpForStream[streamID] = nil // put nil so we don't have to read metadata every time + d.proxyTypeForStream[streamID] = mesh_proto.RegularDpType + return nil } + + if proxyId.ProxyType == mesh_proto.IngressDpType && md.ZoneIngressResource != nil { + lifecycleLog.Info("registering zone ingress", "zoneIngress", md.ZoneIngressResource, "streamID", streamID, "nodeID", request.NodeId()) + if err := d.registerZoneIngress(md.ZoneIngressResource); err != nil { + return errors.Wrap(err, "could not register zone ingress passed in kuma-dp run") + } + key := model.MetaToResourceKey(md.ZoneIngressResource.GetMeta()) + d.createdDpForStream[streamID] = &key + d.proxyTypeForStream[streamID] = mesh_proto.IngressDpType + return nil + } + + d.createdDpForStream[streamID] = nil // put nil so we don't have to read metadata every time return nil } @@ -110,6 +149,18 @@ func (d *DataplaneLifecycle) registerDataplane(dp *core_mesh.DataplaneResource) }) } +func (d *DataplaneLifecycle) registerZoneIngress(zi *core_mesh.ZoneIngressResource) error { + key := model.MetaToResourceKey(zi.GetMeta()) + existing := core_mesh.NewZoneIngressResource() + return manager.Upsert(d.resManager, key, existing, func(resource model.Resource) { + _ = existing.SetSpec(zi.GetSpec()) // ignore error because the spec type is the same + }) +} + func (d *DataplaneLifecycle) unregisterDataplane(key model.ResourceKey) error { return d.resManager.Delete(context.Background(), core_mesh.NewDataplaneResource(), store.DeleteBy(key)) } + +func (d *DataplaneLifecycle) unregisterZoneIngress(key model.ResourceKey) error { + return d.resManager.Delete(context.Background(), core_mesh.NewZoneIngressResource(), store.DeleteBy(key)) +} diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index b1084ff5152a..5d0539b32fef 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -122,7 +122,9 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di statusTrackerLog.Error(err, "failed to extract version out of the Envoy metadata", "streamid", streamID, "metadata", req.Metadata()) } // kick off async Dataplane status flusher - go c.createStatusSink(state).Start(state.stop) + if id.ProxyType != mesh_proto.IngressDpType { + go c.createStatusSink(state).Start(state.stop) + } } else { statusTrackerLog.Error(err, "failed to parse Dataplane Id out of DiscoveryRequest", "streamid", streamID, "req", req) } diff --git a/pkg/xds/server/callbacks/dataplane_sync_tracker.go b/pkg/xds/server/callbacks/dataplane_sync_tracker.go index 1fd65cdbb834..ea6cd1e200df 100644 --- a/pkg/xds/server/callbacks/dataplane_sync_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_sync_tracker.go @@ -4,6 +4,7 @@ import ( "context" stdsync "sync" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_xds "github.com/kumahq/kuma/pkg/core/xds" @@ -15,7 +16,7 @@ var ( dataplaneSyncTrackerLog = core.Log.WithName("xds-server").WithName("dataplane-sync-tracker") ) -type NewDataplaneWatchdogFunc func(dataplaneId core_model.ResourceKey, streamId core_xds.StreamID) util_watchdog.Watchdog +type NewDataplaneWatchdogFunc func(dataplaneId core_model.ResourceKey, streamId core_xds.StreamID, proxyType mesh_proto.DpType) util_watchdog.Watchdog func NewDataplaneSyncTracker(factoryFunc NewDataplaneWatchdogFunc) util_xds.Callbacks { return &dataplaneSyncTracker{ @@ -98,7 +99,7 @@ func (t *dataplaneSyncTracker) OnStreamRequest(streamID core_xds.StreamID, req u close(stopCh) } // kick off watchdog for that Dataplane - go t.newDataplaneWatchdog(dataplaneKey, streamID).Start(stopCh) + go t.newDataplaneWatchdog(dataplaneKey, streamID, id.ProxyType).Start(stopCh) dataplaneSyncTrackerLog.V(1).Info("started Watchdog for a Dataplane", "streamid", streamID, "proxyId", id, "dataplaneKey", dataplaneKey) } t.dpStreams[dataplaneKey] = streams diff --git a/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go b/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go index e131dd6d1892..005303a914cf 100644 --- a/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go +++ b/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go @@ -8,6 +8,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/test" util_watchdog "github.com/kumahq/kuma/pkg/util/watchdog" @@ -79,7 +81,7 @@ var _ = Describe("Sync", func() { watchdogCh := make(chan core_model.ResourceKey) // setup - tracker := NewDataplaneSyncTracker(NewDataplaneWatchdogFunc(func(dataplaneId core_model.ResourceKey, streamId int64) util_watchdog.Watchdog { + tracker := NewDataplaneSyncTracker(NewDataplaneWatchdogFunc(func(dataplaneId core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { return WatchdogFunc(func(stop <-chan struct{}) { watchdogCh <- dataplaneId <-stop @@ -136,7 +138,7 @@ var _ = Describe("Sync", func() { It("should start only one watchdog per dataplane", func() { // setup var activeWatchdogs int32 - tracker := NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64) util_watchdog.Watchdog { + tracker := NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { return WatchdogFunc(func(stop <-chan struct{}) { atomic.AddInt32(&activeWatchdogs, 1) <-stop diff --git a/pkg/xds/server/components.go b/pkg/xds/server/components.go index 0109ee1f513f..4d78c7722215 100644 --- a/pkg/xds/server/components.go +++ b/pkg/xds/server/components.go @@ -26,7 +26,9 @@ var ( core_mesh.ServiceInsightType: true, } HashMeshIncludedGlobalResources = map[core_model.ResourceType]bool{ - core_system.ConfigType: true, + core_system.ConfigType: true, + core_system.GlobalSecretType: true, + core_mesh.ZoneIngressType: true, } ) diff --git a/pkg/xds/sync/dataplane_proxy_builder.go b/pkg/xds/sync/dataplane_proxy_builder.go index c5c3b78eacfb..53484a01b2fc 100644 --- a/pkg/xds/sync/dataplane_proxy_builder.go +++ b/pkg/xds/sync/dataplane_proxy_builder.go @@ -55,7 +55,7 @@ func (p *DataplaneProxyBuilder) build(key core_model.ResourceKey, streamId int64 } proxy := &xds.Proxy{ - Id: xds.FromResourceKey(key), + Id: xds.FromResourceKey(mesh_proto.RegularDpType, key), APIVersion: p.apiVersion, Dataplane: dp, Metadata: p.MetadataTracker.Metadata(streamId), @@ -92,6 +92,11 @@ func (p *DataplaneProxyBuilder) resolveRouting( return nil, nil, err } + zoneIngresses := &core_mesh.ZoneIngressResourceList{} + if err := p.CachingResManager.List(ctx, zoneIngresses); err != nil { + return nil, nil, err + } + matchedExternalServices, err := p.PermissionMatcher.MatchExternalServices(ctx, dataplane, externalServices) if err != nil { return nil, nil, err @@ -107,7 +112,7 @@ func (p *DataplaneProxyBuilder) resolveRouting( destinations := xds_topology.BuildDestinationMap(dataplane, routes) // resolve all endpoints that match given selectors - outbound := xds_topology.BuildEndpointMap(meshContext.Resource, p.Zone, meshContext.Dataplanes.Items, matchedExternalServices, p.DataSourceLoader) + outbound := xds_topology.BuildEndpointMap(meshContext.Resource, p.Zone, meshContext.Dataplanes.Items, zoneIngresses.Items, matchedExternalServices, p.DataSourceLoader) routing := &xds.Routing{ TrafficRoutes: routes, diff --git a/pkg/xds/sync/dataplane_watchdog.go b/pkg/xds/sync/dataplane_watchdog.go index 129b3da642d4..5027007c1f80 100644 --- a/pkg/xds/sync/dataplane_watchdog.go +++ b/pkg/xds/sync/dataplane_watchdog.go @@ -5,12 +5,13 @@ import ( "github.com/go-logr/logr" + mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + "github.com/kumahq/kuma/pkg/core/resources/store" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" - core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" core_model "github.com/kumahq/kuma/pkg/core/resources/model" - core_store "github.com/kumahq/kuma/pkg/core/resources/store" "github.com/kumahq/kuma/pkg/core/xds" "github.com/kumahq/kuma/pkg/xds/cache/mesh" ) @@ -32,25 +33,35 @@ type DataplaneWatchdog struct { log logr.Logger // state of watchdog - lastHash string // last Mesh hash that was used to **successfully** generate Reconcile Envoy config - dpType mesh_proto.DpType + lastHash string // last Mesh hash that was used to **successfully** generate Reconcile Envoy config + dpType mesh_proto.DpType + proxyTypeSettled bool } -func NewDataplaneWatchdog(deps DataplaneWatchdogDependencies, key core_model.ResourceKey, streamId int64) *DataplaneWatchdog { +func NewDataplaneWatchdog(deps DataplaneWatchdogDependencies, key core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) *DataplaneWatchdog { return &DataplaneWatchdog{ DataplaneWatchdogDependencies: deps, key: key, streamId: streamId, log: core.Log.WithValues("key", "key", "streamID", streamId), + dpType: proxyType, + proxyTypeSettled: false, } } func (d *DataplaneWatchdog) Sync() error { - if d.dpType == "" { - // Dataplane type does not change over time therefore we need to figure it once per DataplaneWatchdog - if err := d.inferDpType(); err != nil { + ctx := context.Background() + + // backwards compatibility + if d.dpType == mesh_proto.RegularDpType && !d.proxyTypeSettled { + dataplane := mesh_core.NewDataplaneResource() + if err := d.dataplaneProxyBuilder.CachingResManager.Get(ctx, dataplane, store.GetBy(d.key)); err != nil { return err } + if dataplane.Spec.IsIngress() { + d.dpType = mesh_proto.IngressDpType + } + d.proxyTypeSettled = true } switch d.dpType { case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: @@ -64,7 +75,7 @@ func (d *DataplaneWatchdog) Sync() error { } func (d *DataplaneWatchdog) Cleanup() error { - proxyID := xds.FromResourceKey(d.key) + proxyID := xds.FromResourceKey(d.dpType, d.key) switch d.dpType { case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: return d.dataplaneReconciler.Clear(&proxyID) @@ -75,18 +86,6 @@ func (d *DataplaneWatchdog) Cleanup() error { } } -func (d *DataplaneWatchdog) inferDpType() error { - dataplane := core_mesh.NewDataplaneResource() - if err := d.resManager.Get(context.Background(), dataplane, core_store.GetBy(d.key)); err != nil { - if core_store.IsResourceNotFound(err) { - return nil - } - return err - } - d.dpType = dataplane.Spec.DpType() - return nil -} - // syncDataplane syncs state of the Dataplane. // It uses Mesh Hash to decide if we need to regenerate configuration or not. func (d *DataplaneWatchdog) syncDataplane() error { @@ -118,7 +117,13 @@ func (d *DataplaneWatchdog) syncDataplane() error { // syncIngress synces state of Ingress Dataplane. Notice that it does not use Mesh Hash yet because Ingress supports many Meshes. func (d *DataplaneWatchdog) syncIngress() error { envoyCtx := d.xdsContextBuilder.buildContext(d.streamId) - proxy, err := d.ingressProxyBuilder.build(d.key, d.streamId) + var proxyType mesh_proto.DpType + if d.proxyTypeSettled { + proxyType = mesh_proto.RegularDpType + } else { + proxyType = d.dpType + } + proxy, err := d.ingressProxyBuilder.build(proxyType, d.key, d.streamId) if err != nil { return err } diff --git a/pkg/xds/sync/dataplane_watchdog_factory.go b/pkg/xds/sync/dataplane_watchdog_factory.go index 267c163c7c70..a4dfddcac71f 100644 --- a/pkg/xds/sync/dataplane_watchdog_factory.go +++ b/pkg/xds/sync/dataplane_watchdog_factory.go @@ -3,6 +3,7 @@ package sync import ( "time" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" core_model "github.com/kumahq/kuma/pkg/core/resources/model" util_watchdog "github.com/kumahq/kuma/pkg/util/watchdog" @@ -28,9 +29,9 @@ func NewDataplaneWatchdogFactory( }, nil } -func (d *dataplaneWatchdogFactory) New(key core_model.ResourceKey, streamId int64) util_watchdog.Watchdog { +func (d *dataplaneWatchdogFactory) New(key core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { log := xdsServerLog.WithName("dataplane-sync-watchdog").WithValues("dataplaneKey", key) - dataplaneWatchdog := NewDataplaneWatchdog(d.deps, key, streamId) + dataplaneWatchdog := NewDataplaneWatchdog(d.deps, key, streamId, proxyType) return &util_watchdog.SimpleWatchdog{ NewTicker: func() *time.Ticker { return time.NewTicker(d.refreshInterval) diff --git a/pkg/xds/sync/ingress_proxy_builder.go b/pkg/xds/sync/ingress_proxy_builder.go index 001c36ad0c97..518939fe9996 100644 --- a/pkg/xds/sync/ingress_proxy_builder.go +++ b/pkg/xds/sync/ingress_proxy_builder.go @@ -3,6 +3,8 @@ package sync import ( "context" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" "github.com/kumahq/kuma/pkg/core/dns/lookup" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" @@ -23,10 +25,10 @@ type IngressProxyBuilder struct { apiVersion envoy.APIVersion } -func (p *IngressProxyBuilder) build(key core_model.ResourceKey, streamId int64) (*xds.Proxy, error) { +func (p *IngressProxyBuilder) build(proxyType mesh_proto.DpType, key core_model.ResourceKey, streamId int64) (*xds.Proxy, error) { ctx := context.Background() - dp, err := p.resolveDataplane(ctx, key) + zoneIngress, err := p.getZoneIngress(key) if err != nil { return nil, err } @@ -37,28 +39,54 @@ func (p *IngressProxyBuilder) build(key core_model.ResourceKey, streamId int64) } allMeshDataplanes.Items = xds_topology.ResolveAddresses(syncLog, p.LookupIP, allMeshDataplanes.Items) - // Update Ingress' Available Services - // This was placed as an operation of DataplaneWatchdog out of the convenience. - // Consider moving to the outside of this component (follow the pattern of updating VIP outbounds) - if err := ingress.UpdateAvailableServices(ctx, p.ResManager, dp, allMeshDataplanes.Items); err != nil { - return nil, err - } - - routing, err := p.resolveRouting(ctx, dp, allMeshDataplanes) + routing, err := p.resolveRouting(ctx, zoneIngress, allMeshDataplanes) if err != nil { return nil, err } proxy := &xds.Proxy{ - Id: xds.FromResourceKey(key), - APIVersion: p.apiVersion, - Dataplane: dp, - Metadata: p.MetadataTracker.Metadata(streamId), - Routing: *routing, + Id: xds.FromResourceKey(proxyType, key), + APIVersion: p.apiVersion, + ZoneIngress: zoneIngress, + Metadata: p.MetadataTracker.Metadata(streamId), + Routing: *routing, } return proxy, nil } +func (p *IngressProxyBuilder) getZoneIngress(key core_model.ResourceKey) (*core_mesh.ZoneIngressResource, error) { + ctx := context.Background() + + zoneIngress := core_mesh.NewZoneIngressResource() + err := p.ReadOnlyResManager.Get(ctx, zoneIngress, core_store.GetBy(key)) + if err == nil { + // Update Ingress' Available Services + // This was placed as an operation of DataplaneWatchdog out of the convenience. + // Consider moving to the outside of this component (follow the pattern of updating VIP outbounds) + if err := p.updateIngress(zoneIngress); err != nil { + return nil, err + } + return zoneIngress, nil + } + if !core_store.IsResourceNotFound(err) { + core.Log.WithName("TEST").Error(err, "") + return nil, err + } + + // for backward compatibility with dataplane-based ingresses + oldTypeIngress, err := p.resolveDataplane(ctx, key) + if err != nil { + return nil, err + } + // Update Ingress' Available Services + // This was placed as an operation of DataplaneWatchdog out of the convenience. + // Consider moving to the outside of this component (follow the pattern of updating VIP outbounds) + if err := p.updateIngressCompat(oldTypeIngress); err != nil { + return nil, err + } + return core_mesh.NewZoneIngressResourceFromDataplane(oldTypeIngress) +} + func (p *IngressProxyBuilder) resolveDataplane(ctx context.Context, key core_model.ResourceKey) (*core_mesh.DataplaneResource, error) { dataplane := core_mesh.NewDataplaneResource() @@ -74,8 +102,8 @@ func (p *IngressProxyBuilder) resolveDataplane(ctx context.Context, key core_mod return resolvedDp, nil } -func (p *IngressProxyBuilder) resolveRouting(ctx context.Context, dataplane *core_mesh.DataplaneResource, dataplanes *core_mesh.DataplaneResourceList) (*xds.Routing, error) { - destinations := ingress.BuildDestinationMap(dataplane) +func (p *IngressProxyBuilder) resolveRouting(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, dataplanes *core_mesh.DataplaneResourceList) (*xds.Routing, error) { + destinations := ingress.BuildDestinationMap(zoneIngress) endpoints := ingress.BuildEndpointMap(destinations, dataplanes.Items) routes := &core_mesh.TrafficRouteResourceList{} if err := p.ReadOnlyResManager.List(ctx, routes); err != nil { @@ -88,3 +116,29 @@ func (p *IngressProxyBuilder) resolveRouting(ctx context.Context, dataplane *cor } return routing, nil } + +func (p *IngressProxyBuilder) updateIngressCompat(dpIngress *core_mesh.DataplaneResource) error { + ctx := context.Background() + + allMeshDataplanes := &core_mesh.DataplaneResourceList{} + if err := p.ReadOnlyResManager.List(ctx, allMeshDataplanes); err != nil { + return err + } + allMeshDataplanes.Items = xds_topology.ResolveAddresses(syncLog, p.LookupIP, allMeshDataplanes.Items) + return ingress.UpdateAvailableServicesCompat(ctx, p.ResManager, dpIngress, allMeshDataplanes.Items) +} + +func (p *IngressProxyBuilder) updateIngress(zoneIngress *core_mesh.ZoneIngressResource) error { + ctx := context.Background() + + allMeshDataplanes := &core_mesh.DataplaneResourceList{} + if err := p.ReadOnlyResManager.List(ctx, allMeshDataplanes); err != nil { + return err + } + allMeshDataplanes.Items = xds_topology.ResolveAddresses(syncLog, p.LookupIP, allMeshDataplanes.Items) + + // Update Ingress' Available Services + // This was placed as an operation of DataplaneWatchdog out of the convenience. + // Consider moving to the outside of this component (follow the pattern of updating VIP outbounds) + return ingress.UpdateAvailableServices(ctx, p.ResManager, zoneIngress, allMeshDataplanes.Items) +} diff --git a/pkg/xds/sync/interfaces.go b/pkg/xds/sync/interfaces.go index 4c5c2f5b0b97..0418d560356f 100644 --- a/pkg/xds/sync/interfaces.go +++ b/pkg/xds/sync/interfaces.go @@ -1,6 +1,7 @@ package sync import ( + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_xds "github.com/kumahq/kuma/pkg/core/xds" util_watchdog "github.com/kumahq/kuma/pkg/util/watchdog" @@ -23,5 +24,5 @@ type SnapshotReconciler interface { // DataplaneWatchdogFactory returns a Watchdog that creates a new XdsContext and Proxy and executes SnapshotReconciler if there is any change type DataplaneWatchdogFactory interface { - New(key core_model.ResourceKey, streamId int64) util_watchdog.Watchdog + New(key core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog } diff --git a/pkg/xds/topology/outbound.go b/pkg/xds/topology/outbound.go index 42786c2ebf79..afc3ce4f11f7 100644 --- a/pkg/xds/topology/outbound.go +++ b/pkg/xds/topology/outbound.go @@ -27,10 +27,11 @@ func BuildEndpointMap( mesh *mesh_core.MeshResource, zone string, dataplanes []*mesh_core.DataplaneResource, + zoneIngresses []*mesh_core.ZoneIngressResource, externalServices []*mesh_core.ExternalServiceResource, loader datasource.Loader, ) core_xds.EndpointMap { - outbound := BuildEdsEndpointMap(mesh, zone, dataplanes) + outbound := BuildEdsEndpointMap(mesh, zone, dataplanes, zoneIngresses) fillExternalServicesOutbounds(outbound, externalServices, mesh, loader) return outbound } @@ -39,9 +40,10 @@ func BuildEdsEndpointMap( mesh *mesh_core.MeshResource, zone string, dataplanes []*mesh_core.DataplaneResource, + zoneIngresses []*mesh_core.ZoneIngressResource, ) core_xds.EndpointMap { outbound := core_xds.EndpointMap{} - ingressInstances := fillIngressOutbounds(outbound, dataplanes, zone, mesh) + ingressInstances := fillIngressOutbounds(outbound, zoneIngresses, dataplanes, zone, mesh) endpointWeight := uint32(1) if ingressInstances > 0 { endpointWeight = ingressInstances @@ -91,8 +93,52 @@ func fillDataplaneOutbounds(outbound core_xds.EndpointMap, dataplanes []*mesh_co } } -func fillIngressOutbounds(outbound core_xds.EndpointMap, dataplanes []*mesh_core.DataplaneResource, zone string, mesh *mesh_core.MeshResource) uint32 { +func fillIngressOutbounds( + outbound core_xds.EndpointMap, + zoneIngresses []*mesh_core.ZoneIngressResource, + dataplanes []*mesh_core.DataplaneResource, + zone string, + mesh *mesh_core.MeshResource, +) uint32 { ingressInstances := map[string]bool{} + core.Log.WithName("TEST").Info("fillIngressOutbounds", "zoneIngresses", zoneIngresses) + + for _, zi := range zoneIngresses { + core.Log.WithName("TEST").Info("fillIngressOutbounds", "zi", zi) + if !zi.IsRemoteIngress(zone) { + core.Log.WithName("TEST").Info("fillIngressOutbounds, not remote") + continue + } + if !mesh.MTLSEnabled() { + // Ingress routes the request by TLS SNI, therefore for cross cluster communication MTLS is required + // We ignore Ingress from endpoints if MTLS is disabled, otherwise we would fail anyway. + continue + } + if !zi.HasPublicAddress() { + continue // Zone Ingress is not reachable yet from other clusters. This may happen when Ingress Service is pending waiting on External IP on Kubernetes. + } + ingressCoordinates := net.JoinHostPort(zi.Spec.GetAdvertisedAddress(), strconv.FormatUint(uint64(zi.Spec.GetAdvertisedPort()), 10)) + if ingressInstances[ingressCoordinates] { + continue // many Ingress instances can be placed in front of one load balancer (all instances can have the same public address and port). In this case we only need one Instance avoiding creating unnecessary duplicated endpoints + } + core.Log.WithName("TEST").Info("fillIngressOutbounds", "zi", zi, "len", len(zi.Spec.AvailableServices)) + for _, service := range zi.Spec.GetAvailableServices() { + core.Log.WithName("TEST").Info("fillIngressOutbounds", "zi", zi, "availableService", service) + if service.Mesh != mesh.GetMeta().GetName() { + continue + } + serviceName := service.Tags[mesh_proto.ServiceTag] + outbound[serviceName] = append(outbound[serviceName], core_xds.Endpoint{ + Target: zi.Spec.GetAdvertisedAddress(), + Port: zi.Spec.GetAdvertisedPort(), + Tags: service.Tags, + Weight: service.Instances, + Locality: localityFromTags(mesh, priorityRemote, service.Tags), + }) + } + } + + // backwards compatibility for _, dataplane := range dataplanes { if !dataplane.Spec.IsIngress() { continue diff --git a/pkg/xds/topology/outbound_test.go b/pkg/xds/topology/outbound_test.go index 4c806db590fa..7b9b1860e09f 100644 --- a/pkg/xds/topology/outbound_test.go +++ b/pkg/xds/topology/outbound_test.go @@ -167,7 +167,7 @@ var _ = Describe("TrafficRoute", func() { externalServices := &mesh_core.ExternalServiceResourceList{} // when - targets := BuildEndpointMap(defaultMeshWithMTLS, "zone-1", dataplanes.Items, externalServices.Items, dataSourceLoader) + targets := BuildEndpointMap(defaultMeshWithMTLS, "zone-1", dataplanes.Items, nil, externalServices.Items, dataSourceLoader) Expect(targets).To(HaveLen(4)) // and @@ -252,7 +252,7 @@ var _ = Describe("TrafficRoute", func() { DescribeTable("should include only those dataplanes that match given selectors", func(given testCase) { // when - endpoints := BuildEndpointMap(given.mesh, "zone-1", given.dataplanes, given.externalServices, dataSourceLoader) + endpoints := BuildEndpointMap(given.mesh, "zone-1", given.dataplanes, nil, given.externalServices, dataSourceLoader) // then Expect(endpoints).To(Equal(given.expected)) }, diff --git a/test/e2e/deploy/kuma_deploy_universal.go b/test/e2e/deploy/kuma_deploy_universal.go index 49617315ce6e..5c721fc9084b 100644 --- a/test/e2e/deploy/kuma_deploy_universal.go +++ b/test/e2e/deploy/kuma_deploy_universal.go @@ -66,7 +66,7 @@ name: %s Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken(nonDefaultMesh, "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateDpToken(defaultMesh, "ingress") + ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") Expect(err).ToNot(HaveOccurred()) // TODO: right now these tests are deliberately run WithHDS(false) @@ -83,7 +83,7 @@ name: %s Install(Kuma(core.Remote, optsRemote1...)). Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal1", echoServerToken, WithTransparentProxy(true))). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(defaultMesh, ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remote_1) Expect(err).ToNot(HaveOccurred()) err = remote_1.VerifyKuma() @@ -99,7 +99,7 @@ name: %s Install(Kuma(core.Remote, optsRemote2...)). Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal2", echoServerToken, WithTransparentProxy(true))). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(defaultMesh, ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remote_2) Expect(err).ToNot(HaveOccurred()) err = remote_2.VerifyKuma() diff --git a/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go b/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go index ac9cc7737b01..d6e66e8750f0 100644 --- a/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go +++ b/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go @@ -75,7 +75,7 @@ metadata: echoServerToken, err := global.GetKuma().GenerateDpToken("default", "echo-server_kuma-test_svc_8080") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := global.GetKuma().GenerateDpToken("default", "ingress") + ingressToken, err := global.GetKuma().GenerateZoneIngressToken("ingress") Expect(err).ToNot(HaveOccurred()) optsRemoteUniversal = append(optsRemoteUniversal, @@ -87,7 +87,7 @@ metadata: Install(EchoServerUniversal("dp-echo-1", "default", "echo-universal-1", echoServerToken, WithProtocol("tcp"))). Install(EchoServerUniversal("dp-echo-2", "default", "echo-universal-2", echoServerToken, WithProtocol("tcp"), ProxyOnly(), ServiceProbe())). Install(EchoServerUniversal("dp-echo-3", "default", "echo-universal-3", echoServerToken, WithProtocol("tcp"))). - Install(IngressUniversal("default", ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remoteUniversal) Expect(err).ToNot(HaveOccurred()) err = remoteUniversal.VerifyKuma() diff --git a/test/e2e/hybrid/kuma_hybrid.go b/test/e2e/hybrid/kuma_hybrid.go index f1e93b2e9089..7f9664480c62 100644 --- a/test/e2e/hybrid/kuma_hybrid.go +++ b/test/e2e/hybrid/kuma_hybrid.go @@ -77,7 +77,7 @@ metadata: Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken(nonDefaultMesh, "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateDpToken(defaultMesh, "ingress") + ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") Expect(err).ToNot(HaveOccurred()) // K8s Cluster 1 @@ -124,7 +124,7 @@ metadata: Install(Kuma(core.Remote, optsRemote3...)). Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal", echoServerToken, WithTransparentProxy(true))). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(defaultMesh, ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remote_3) Expect(err).ToNot(HaveOccurred()) err = remote_3.VerifyKuma() @@ -138,7 +138,7 @@ metadata: err = NewClusterSetup(). Install(Kuma(core.Remote, optsRemote4...)). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken)). - Install(IngressUniversal(defaultMesh, ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remote_4) Expect(err).ToNot(HaveOccurred()) err = remote_4.VerifyKuma() diff --git a/test/e2e/hybrid/kuma_hybrid_kube_global.go b/test/e2e/hybrid/kuma_hybrid_kube_global.go index 843b8f0f8362..72291d4071d5 100644 --- a/test/e2e/hybrid/kuma_hybrid_kube_global.go +++ b/test/e2e/hybrid/kuma_hybrid_kube_global.go @@ -42,7 +42,7 @@ func KubernetesUniversalDeploymentWhenGlobalIsOnK8S() { Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken("default", "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateDpToken("default", "ingress") + ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") Expect(err).ToNot(HaveOccurred()) // Remote @@ -54,7 +54,7 @@ func KubernetesUniversalDeploymentWhenGlobalIsOnK8S() { Install(Kuma(core.Remote, optsRemote...)). Install(EchoServerUniversal(AppModeEchoServer, "default", "universal", echoServerToken)). Install(DemoClientUniversal(AppModeDemoClient, "default", demoClientToken)). - Install(IngressUniversal("default", ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remoteCluster) Expect(err).ToNot(HaveOccurred()) err = remoteCluster.VerifyKuma() diff --git a/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go b/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go index 1bf4ddb590d4..0f58f1a3d84a 100644 --- a/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go +++ b/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go @@ -69,7 +69,7 @@ spec: echoServerToken, err := globalCP.GenerateDpToken("default", "echo-server_kuma-test_svc_8080") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateDpToken("default", "ingress") + ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") Expect(err).ToNot(HaveOccurred()) // Remote universal @@ -80,7 +80,7 @@ spec: err = NewClusterSetup(). Install(Kuma(config_core.Remote, optsRemoteUniversal...)). Install(EchoServerUniversal(AppModeEchoServer, "default", "universal", echoServerToken)). - Install(IngressUniversal("default", ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remoteUniversal) Expect(err).ToNot(HaveOccurred()) err = remoteUniversal.VerifyKuma() diff --git a/test/e2e/trafficroute/universal_multizone/traffic_route.go b/test/e2e/trafficroute/universal_multizone/traffic_route.go index 83d039afbb82..7f4c9ebab3da 100644 --- a/test/e2e/trafficroute/universal_multizone/traffic_route.go +++ b/test/e2e/trafficroute/universal_multizone/traffic_route.go @@ -56,7 +56,7 @@ routing: Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken(defaultMesh, "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateDpToken(defaultMesh, "ingress") + ingressToken, err := globalCP.GenerateZoneIngressToken("new-ingress") Expect(err).ToNot(HaveOccurred()) // Cluster 1 @@ -68,7 +68,7 @@ routing: err = NewClusterSetup(). Install(Kuma(core.Remote, optsRemote1...)). Install(DemoClientUniversal(AppModeDemoClient, defaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(defaultMesh, ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remote_1) Expect(err).ToNot(HaveOccurred()) err = remote_1.VerifyKuma() @@ -103,7 +103,7 @@ routing: WithServiceVersion("v1"), WithTransparentProxy(true), )). - Install(IngressUniversal(defaultMesh, ingressToken)). + Install(IngressUniversal(ingressToken)). Setup(remote_2) Expect(err).ToNot(HaveOccurred()) err = remote_2.VerifyKuma() diff --git a/test/framework/interface.go b/test/framework/interface.go index 7307bfda40db..802bac46d1f3 100644 --- a/test/framework/interface.go +++ b/test/framework/interface.go @@ -300,4 +300,5 @@ type ControlPlane interface { GetKDSServerAddress() string GetGlobaStatusAPI() string GenerateDpToken(mesh, appname string) (string, error) + GenerateZoneIngressToken(name string) (string, error) } diff --git a/test/framework/k8s_controlplane.go b/test/framework/k8s_controlplane.go index a510ab0e9c73..a09db9cef3ba 100644 --- a/test/framework/k8s_controlplane.go +++ b/test/framework/k8s_controlplane.go @@ -268,3 +268,17 @@ func (c *K8sControlPlane) GenerateDpToken(mesh, service string) (string, error) &tls.Config{}, ) } + +func (c *K8sControlPlane) GenerateZoneIngressToken(name string) (string, error) { + return http_helper.HTTPDoWithRetryE( + c.t, + "POST", + fmt.Sprintf("http://localhost:%d/tokens/zone-ingress", c.portFwd.localAPIPort), + []byte(fmt.Sprintf(`{"name": "%s"}`, name)), + map[string]string{"content-type": "application/json"}, + 200, + DefaultRetries, + DefaultTimeout, + &tls.Config{}, + ) +} diff --git a/test/framework/setup.go b/test/framework/setup.go index dedc7a304ce2..94ca6c90a734 100644 --- a/test/framework/setup.go +++ b/test/framework/setup.go @@ -276,7 +276,7 @@ func EchoServerUniversal(name, mesh, echo, token string, fs ...DeployOptionsFunc } } -func IngressUniversal(mesh, token string) InstallFunc { +func IngressUniversalOldType(mesh, token string) InstallFunc { return func(cluster Cluster) error { uniCluster := cluster.(*UniversalCluster) isipv6 := IsIPv6() @@ -295,11 +295,35 @@ func IngressUniversal(mesh, token string) InstallFunc { uniCluster.apps[AppIngress] = app publicAddress := uniCluster.apps[AppIngress].ip - dpyaml := fmt.Sprintf(IngressDataplane, mesh, publicAddress, kdsPort, kdsPort) + dpyaml := fmt.Sprintf(IngressDataplaneOldType, mesh, publicAddress, kdsPort, kdsPort) return uniCluster.CreateDP(app, "ingress", app.ip, dpyaml, token, false) } } +func IngressUniversal(token string) InstallFunc { + return func(cluster Cluster) error { + uniCluster := cluster.(*UniversalCluster) + isipv6 := IsIPv6() + verbose := false + app, err := NewUniversalApp(cluster.GetTesting(), uniCluster.name, AppIngress, AppIngress, isipv6, verbose, []string{}) + if err != nil { + return err + } + + app.CreateMainApp([]string{}, []string{}) + + err = app.mainApp.Start() + if err != nil { + return err + } + uniCluster.apps[AppIngress] = app + + publicAddress := uniCluster.apps[AppIngress].ip + dpyaml := fmt.Sprintf(ZoneIngress, publicAddress, kdsPort, kdsPort) + return uniCluster.CreateZoneIngress(app, "ingress", app.ip, dpyaml, token, false) + } +} + func DemoClientK8s(mesh string) InstallFunc { const name = "demo-client" deployment := ` diff --git a/test/framework/universal_app.go b/test/framework/universal_app.go index c34eacec2c33..816688a9f8eb 100644 --- a/test/framework/universal_app.go +++ b/test/framework/universal_app.go @@ -29,7 +29,7 @@ const ( AppModeHttpsEchoServer = "https-echo-server" sshPort = "22" - IngressDataplane = ` + IngressDataplaneOldType = ` type: Dataplane mesh: %s name: dp-ingress @@ -43,6 +43,15 @@ networking: tags: kuma.io/service: ingress ` + ZoneIngress = ` +type: ZoneIngress +name: new-ingress +address: {{ address }} +advertisedAddress: %s +advertisedPort: %d +port: %d +` + EchoServerDataplane = ` type: Dataplane mesh: %s @@ -354,7 +363,7 @@ func (s *UniversalApp) OverrideDpVersion(version string) error { return nil } -func (s *UniversalApp) CreateDP(token, cpAddress, appname, ip, dpyaml string, builtindns bool) { +func (s *UniversalApp) CreateDP(token, cpAddress, appname, ip, dpyaml string, builtindns, ingress bool) { // create the token file on the app container err := NewSshApp(s.verbose, s.ports[sshPort], []string{}, []string{"printf ", "\"" + token + "\"", ">", "/kuma/token-" + appname}).Run() if err != nil { @@ -380,6 +389,9 @@ func (s *UniversalApp) CreateDP(token, cpAddress, appname, ip, dpyaml string, bu if builtindns { args = append(args, "--dns-enabled") } + if ingress { + args = append(args, "--ingress") + } s.dpApp = NewSshApp(s.verbose, s.ports[sshPort], []string{}, args) } diff --git a/test/framework/universal_cluster.go b/test/framework/universal_cluster.go index a89e3e4d19ba..9b1384fcd4b4 100644 --- a/test/framework/universal_cluster.go +++ b/test/framework/universal_cluster.go @@ -167,7 +167,14 @@ func (c *UniversalCluster) DeleteNamespace(namespace string) error { func (c *UniversalCluster) CreateDP(app *UniversalApp, appname, ip, dpyaml, token string, builtindns bool) error { cpIp := c.apps[AppModeCP].ip cpAddress := "https://" + net.JoinHostPort(cpIp, "5678") - app.CreateDP(token, cpAddress, appname, ip, dpyaml, builtindns) + app.CreateDP(token, cpAddress, appname, ip, dpyaml, builtindns, false) + return app.dpApp.Start() +} + +func (c *UniversalCluster) CreateZoneIngress(app *UniversalApp, appname, ip, dpyaml, token string, builtindns bool) error { + cpIp := c.apps[AppModeCP].ip + cpAddress := "https://" + net.JoinHostPort(cpIp, "5678") + app.CreateDP(token, cpAddress, appname, ip, dpyaml, builtindns, true) return app.dpApp.Start() } diff --git a/test/framework/universal_clusters.go b/test/framework/universal_clusters.go index bbb1d744d849..867c388d2a79 100644 --- a/test/framework/universal_clusters.go +++ b/test/framework/universal_clusters.go @@ -25,7 +25,11 @@ func NewUniversalClusters(clusterNames []string, verbose bool) (Clusters, error) clusters := map[string]*UniversalCluster{} for _, name := range clusterNames { - clusters[name] = NewUniversalCluster(t, name, verbose) + if name == Kuma4 { + clusters[name] = NewUniversalCluster(t, name, true) + } else { + clusters[name] = NewUniversalCluster(t, name, false) + } } return &UniversalClusters{ diff --git a/test/framework/universal_controlplane.go b/test/framework/universal_controlplane.go index 5074d00de620..cd232161d9ef 100644 --- a/test/framework/universal_controlplane.go +++ b/test/framework/universal_controlplane.go @@ -73,3 +73,20 @@ func (c *UniversalControlPlane) GenerateDpToken(mesh, service string) (string, e return sshApp.Out(), nil }) } + +func (c *UniversalControlPlane) GenerateZoneIngressToken(name string) (string, error) { + return retry.DoWithRetryE(c.t, "generating DP token", DefaultRetries, DefaultTimeout, func() (string, error) { + sshApp := NewSshApp(c.verbose, c.cluster.apps[AppModeCP].ports["22"], []string{}, []string{"curl", + "--fail", "--show-error", + "-H", "\"Content-Type: application/json\"", + "--data", fmt.Sprintf(`'{"name": "%s"}'`, name), + "http://localhost:5681/tokens/zone-ingress"}) + if err := sshApp.Run(); err != nil { + return "", err + } + if sshApp.Err() != "" { + return "", errors.New(sshApp.Err()) + } + return sshApp.Out(), nil + }) +} From 34327a69258e6d73d3fdb6731e9b929f8c4845d9 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Thu, 10 Jun 2021 19:01:20 +0700 Subject: [PATCH 03/19] chore(kuma-cp) make check Signed-off-by: Ilya Lobkov --- pkg/sds/server/v2/reconciler.go | 0 pkg/sds/server/v2/server.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 pkg/sds/server/v2/reconciler.go delete mode 100644 pkg/sds/server/v2/server.go diff --git a/pkg/sds/server/v2/reconciler.go b/pkg/sds/server/v2/reconciler.go deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pkg/sds/server/v2/server.go b/pkg/sds/server/v2/server.go deleted file mode 100644 index e69de29bb2d1..000000000000 From 146e3c881c3571fcf6282f499bbd1e1f2f541a90 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Thu, 10 Jun 2021 20:13:13 +0700 Subject: [PATCH 04/19] chore(kuma-cp) resolve ingress address Signed-off-by: Ilya Lobkov --- pkg/xds/cache/cla/cache.go | 5 +-- pkg/xds/cache/mesh/snapshot.go | 9 +++++ pkg/xds/sync/dataplane_proxy_builder.go | 4 +-- pkg/xds/sync/ingress_proxy_builder.go | 4 +++ pkg/xds/topology/dataplanes.go | 45 +++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 4 deletions(-) diff --git a/pkg/xds/cache/cla/cache.go b/pkg/xds/cache/cla/cache.go index 9354ef611a81..071a8aaa4156 100644 --- a/pkg/xds/cache/cla/cache.go +++ b/pkg/xds/cache/cla/cache.go @@ -62,10 +62,11 @@ func (c *Cache) GetCLA(ctx context.Context, meshName, meshHash string, cluster e if err != nil { return nil, err } - zoneIngresses := &core_mesh.ZoneIngressResourceList{} - if err := c.rm.List(ctx, zoneIngresses); err != nil { + zoneIngresses, err := topology.GetZoneIngresses(claCacheLog, ctx, c.rm, c.ipFunc) + if err != nil { return nil, err } + mesh := core_mesh.NewMeshResource() if err := c.rm.Get(ctx, mesh, core_store.GetByKey(meshName, model.NoMesh)); err != nil { return nil, err diff --git a/pkg/xds/cache/mesh/snapshot.go b/pkg/xds/cache/mesh/snapshot.go index e5a95b5c2004..79eac29544e1 100644 --- a/pkg/xds/cache/mesh/snapshot.go +++ b/pkg/xds/cache/mesh/snapshot.go @@ -126,6 +126,15 @@ func (m *meshSnapshot) hashResource(r core_model.Resource) string { m.hashResolvedIPs(v.Spec.GetNetworking().GetAddress()), m.hashResolvedIPs(v.Spec.GetNetworking().GetIngress().GetPublicAddress()), }, ":") + case *core_mesh.ZoneIngressResource: + return strings.Join( + []string{string(v.GetType()), + v.GetMeta().GetMesh(), + v.GetMeta().GetName(), + v.GetMeta().GetVersion(), + m.hashResolvedIPs(v.Spec.GetAddress()), + m.hashResolvedIPs(v.Spec.GetAdvertisedAddress()), + }, ":") default: return strings.Join( []string{string(v.GetType()), diff --git a/pkg/xds/sync/dataplane_proxy_builder.go b/pkg/xds/sync/dataplane_proxy_builder.go index 53484a01b2fc..c73d2e591a9a 100644 --- a/pkg/xds/sync/dataplane_proxy_builder.go +++ b/pkg/xds/sync/dataplane_proxy_builder.go @@ -92,8 +92,8 @@ func (p *DataplaneProxyBuilder) resolveRouting( return nil, nil, err } - zoneIngresses := &core_mesh.ZoneIngressResourceList{} - if err := p.CachingResManager.List(ctx, zoneIngresses); err != nil { + zoneIngresses, err := xds_topology.GetZoneIngresses(syncLog, ctx, p.CachingResManager, p.LookupIP) + if err != nil { return nil, nil, err } diff --git a/pkg/xds/sync/ingress_proxy_builder.go b/pkg/xds/sync/ingress_proxy_builder.go index 518939fe9996..05a66299fe1f 100644 --- a/pkg/xds/sync/ingress_proxy_builder.go +++ b/pkg/xds/sync/ingress_proxy_builder.go @@ -32,6 +32,10 @@ func (p *IngressProxyBuilder) build(proxyType mesh_proto.DpType, key core_model. if err != nil { return nil, err } + zoneIngress, err = xds_topology.ResolveZoneIngressPublicAddress(p.LookupIP, zoneIngress) + if err != nil { + return nil, err + } allMeshDataplanes := &core_mesh.DataplaneResourceList{} if err := p.ReadOnlyResManager.List(ctx, allMeshDataplanes); err != nil { diff --git a/pkg/xds/topology/dataplanes.go b/pkg/xds/topology/dataplanes.go index e7a2acb87330..be0f19a5f95b 100644 --- a/pkg/xds/topology/dataplanes.go +++ b/pkg/xds/topology/dataplanes.go @@ -24,6 +24,7 @@ func GetDataplanes(log logr.Logger, ctx context.Context, rm manager.ReadOnlyReso dataplanes.Items = ResolveAddresses(log, lookupIPFunc, dataplanes.Items) filteredDataplanes := &core_mesh.DataplaneResourceList{} for _, d := range dataplanes.Items { + // backwards compatibility if d.GetMeta().GetMesh() == mesh || d.Spec.IsIngress() { _ = filteredDataplanes.AddItem(d) } @@ -32,6 +33,15 @@ func GetDataplanes(log logr.Logger, ctx context.Context, rm manager.ReadOnlyReso return filteredDataplanes, nil } +func GetZoneIngresses(log logr.Logger, ctx context.Context, rm manager.ReadOnlyResourceManager, lookupIPFunc lookup.LookupIPFunc) (*core_mesh.ZoneIngressResourceList, error) { + zoneIngresses := &core_mesh.ZoneIngressResourceList{} + if err := rm.List(ctx, zoneIngresses); err != nil { + return nil, err + } + zoneIngresses.Items = ResolveZoneIngressAddresses(log, lookupIPFunc, zoneIngresses.Items) + return zoneIngresses, nil +} + // ResolveAddress resolves 'dataplane.networking.address' if it has DNS name in it. This is a crucial feature for // some environments specifically AWS ECS. Dataplane resource has to be created before running Kuma DP, but IP address // will be assigned only after container's start. Envoy EDS doesn't support DNS names, that's why Kuma CP resolves @@ -77,6 +87,28 @@ func ResolveIngressPublicAddress(lookupIPFunc lookup.LookupIPFunc, dataplane *co return dataplane, nil } +func ResolveZoneIngressPublicAddress(lookupIPFunc lookup.LookupIPFunc, zoneIngress *core_mesh.ZoneIngressResource) (*core_mesh.ZoneIngressResource, error) { + if zoneIngress.Spec.GetAdvertisedAddress() == "" { // Ingress may not have public address yet. + return zoneIngress, nil + } + ips, err := lookupIPFunc(zoneIngress.Spec.GetAdvertisedAddress()) + if err != nil { + return nil, err + } + if len(ips) == 0 { + return nil, errors.Errorf("can't resolve address %v", zoneIngress.Spec.GetAdvertisedAddress()) + } + if zoneIngress.Spec.GetAdvertisedAddress() != ips[0].String() { // only if we resolve any address, in most cases this is IP not a hostname + ziSpec := proto.Clone(zoneIngress.Spec).(*mesh_proto.ZoneIngress) + ziSpec.AdvertisedAddress = ips[0].String() + return &core_mesh.ZoneIngressResource{ + Meta: zoneIngress.Meta, + Spec: ziSpec, + }, nil + } + return zoneIngress, nil +} + func ResolveAddresses(log logr.Logger, lookupIPFunc lookup.LookupIPFunc, dataplanes []*core_mesh.DataplaneResource) []*core_mesh.DataplaneResource { rv := []*core_mesh.DataplaneResource{} for _, d := range dataplanes { @@ -98,3 +130,16 @@ func ResolveAddresses(log logr.Logger, lookupIPFunc lookup.LookupIPFunc, datapla } return rv } + +func ResolveZoneIngressAddresses(log logr.Logger, lookupIPFunc lookup.LookupIPFunc, zoneIngresses []*core_mesh.ZoneIngressResource) []*core_mesh.ZoneIngressResource { + rv := []*core_mesh.ZoneIngressResource{} + for _, zi := range zoneIngresses { + resolvedZi, err := ResolveZoneIngressPublicAddress(lookupIPFunc, zi) + if err != nil { + log.Error(err, "failed to resolve ingress's public name, skipping dataplane") + continue + } + rv = append(rv, resolvedZi) + } + return rv +} From b3a780ca5b60087b27e1275addeb8d9908e8b672 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Thu, 10 Jun 2021 20:30:06 +0700 Subject: [PATCH 05/19] chore(kuma-cp) fix test, get rid of debug logs Signed-off-by: Ilya Lobkov --- pkg/kds/zone/components.go | 1 - pkg/xds/auth/callbacks.go | 2 -- pkg/xds/bootstrap/generator.go | 13 +++++++------ pkg/xds/sync/ingress_proxy_builder.go | 2 -- pkg/xds/topology/outbound.go | 5 ----- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/pkg/kds/zone/components.go b/pkg/kds/zone/components.go index 85aea79f2949..1404f0a9cb91 100644 --- a/pkg/kds/zone/components.go +++ b/pkg/kds/zone/components.go @@ -111,7 +111,6 @@ func Callbacks(rt core_runtime.Runtime, syncer sync_store.ResourceSyncer, k8sSto })) } if rs.GetItemType() == mesh.ZoneIngressType { - core.Log.WithName("TEST").Info("OnResourcesReceived ZoneIngress", "resources", rs) return syncer.Sync(rs, sync_store.PrefilterBy(func(r model.Resource) bool { return r.(*mesh.ZoneIngressResource).IsRemoteIngress(localZone) })) diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index 05e6b225f029..40436b92f434 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -10,7 +10,6 @@ import ( "google.golang.org/grpc/metadata" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - "github.com/kumahq/kuma/pkg/core" core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" @@ -89,7 +88,6 @@ func (a *authCallbacks) OnStreamRequest(streamID core_xds.StreamID, req util_xds return err } err = a.authenticate(credential, req) - core.Log.WithName("TEST").Error(err, "auth.OnStreamRequest") if err != nil { return err } diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 7134eb8349ef..dee7bd57a217 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -17,8 +17,6 @@ import ( "github.com/asaskevich/govalidator" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - "github.com/kumahq/kuma/pkg/core" - core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/resources/model/rest" "github.com/kumahq/kuma/pkg/core/validators" @@ -80,7 +78,12 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr return nil, "", err } - switch mesh_proto.DpType(request.ProxyType) { + proxyType := mesh_proto.DpType(request.ProxyType) + if request.ProxyType == "" { + proxyType = mesh_proto.RegularDpType + } + + switch proxyType { case mesh_proto.IngressDpType: proxyId := core_xds.BuildProxyId(request.Mesh, request.Name, mesh_proto.IngressDpType) zoneIngress, err := b.zoneIngressFor(ctx, request, proxyId) @@ -106,7 +109,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr } return b.generateFor(*proxyId, request, service, adminPort) default: - return nil, "", nil + return nil, "", errors.Errorf("unknown proxy type %v", proxyType) } } @@ -205,9 +208,7 @@ func (b *bootstrapGenerator) zoneIngressFor(ctx context.Context, request types.B return zoneIngress, nil } else { zoneIngress := core_mesh.NewZoneIngressResource() - core.Log.WithName("TEST").Info("zoneIngressFor", "key", proxyId.Name) if err := b.resManager.Get(ctx, zoneIngress, core_store.GetByKey(proxyId.Name, core_model.NoMesh)); err != nil { - core.Log.WithName("TEST").Error(err, "zoneIngressFor") return nil, err } return zoneIngress, nil diff --git a/pkg/xds/sync/ingress_proxy_builder.go b/pkg/xds/sync/ingress_proxy_builder.go index 05a66299fe1f..d897af051de3 100644 --- a/pkg/xds/sync/ingress_proxy_builder.go +++ b/pkg/xds/sync/ingress_proxy_builder.go @@ -4,7 +4,6 @@ import ( "context" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - "github.com/kumahq/kuma/pkg/core" "github.com/kumahq/kuma/pkg/core/dns/lookup" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" @@ -73,7 +72,6 @@ func (p *IngressProxyBuilder) getZoneIngress(key core_model.ResourceKey) (*core_ return zoneIngress, nil } if !core_store.IsResourceNotFound(err) { - core.Log.WithName("TEST").Error(err, "") return nil, err } diff --git a/pkg/xds/topology/outbound.go b/pkg/xds/topology/outbound.go index 645597e9f0b3..31dbff00240c 100644 --- a/pkg/xds/topology/outbound.go +++ b/pkg/xds/topology/outbound.go @@ -101,12 +101,9 @@ func fillIngressOutbounds( mesh *mesh_core.MeshResource, ) uint32 { ingressInstances := map[string]bool{} - core.Log.WithName("TEST").Info("fillIngressOutbounds", "zoneIngresses", zoneIngresses) for _, zi := range zoneIngresses { - core.Log.WithName("TEST").Info("fillIngressOutbounds", "zi", zi) if !zi.IsRemoteIngress(zone) { - core.Log.WithName("TEST").Info("fillIngressOutbounds, not remote") continue } if !mesh.MTLSEnabled() { @@ -121,9 +118,7 @@ func fillIngressOutbounds( if ingressInstances[ingressCoordinates] { continue // many Ingress instances can be placed in front of one load balancer (all instances can have the same public address and port). In this case we only need one Instance avoiding creating unnecessary duplicated endpoints } - core.Log.WithName("TEST").Info("fillIngressOutbounds", "zi", zi, "len", len(zi.Spec.AvailableServices)) for _, service := range zi.Spec.GetAvailableServices() { - core.Log.WithName("TEST").Info("fillIngressOutbounds", "zi", zi, "availableService", service) if service.Mesh != mesh.GetMeta().GetName() { continue } From 67e932bf1892b2f3d6833f31a9e047b32d0b813b Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Thu, 10 Jun 2021 22:08:19 +0700 Subject: [PATCH 06/19] chore(kuma-cp) pass ProxyId everywhere Signed-off-by: Ilya Lobkov --- pkg/core/xds/types.go | 57 +++++++++++-------- pkg/core/xds/types_test.go | 29 +++------- pkg/hds/authn/callbacks.go | 2 +- pkg/hds/tracker/callbacks.go | 6 +- pkg/hds/tracker/healthcheck_generator.go | 6 +- pkg/sds/server/v3/reconciler.go | 33 +++++------ pkg/sds/server/v3/server.go | 11 ++-- pkg/xds/auth/callbacks.go | 8 +-- pkg/xds/bootstrap/generator.go | 2 +- .../v3/http_access_log_configurer_test.go | 6 +- .../v3/network_access_log_configurer_test.go | 6 +- pkg/xds/generator/dns_generator_test.go | 2 +- .../generator/inbound_proxy_generator_test.go | 2 +- pkg/xds/generator/ingress_generator_test.go | 2 +- .../outbound_proxy_generator_test.go | 4 +- .../prometheus_endpoint_generator_test.go | 20 +++---- .../proxy_template_profile_source_test.go | 2 +- .../proxy_template_raw_source_test.go | 18 +++--- pkg/xds/generator/proxy_template_test.go | 4 +- pkg/xds/generator/tracing_generator_test.go | 4 +- .../transparent_proxy_generator_test.go | 8 +-- .../server/callbacks/dataplane_lifecycle.go | 4 +- .../callbacks/dataplane_status_tracker.go | 6 +- .../callbacks/dataplane_sync_tracker.go | 11 ++-- .../callbacks/dataplane_sync_tracker_test.go | 10 ++-- pkg/xds/server/v3/reconcile_test.go | 7 +-- pkg/xds/server/v3/snapshot_generator_test.go | 2 +- pkg/xds/sync/dataplane_watchdog.go | 10 ++-- pkg/xds/sync/dataplane_watchdog_factory.go | 9 ++- pkg/xds/sync/interfaces.go | 4 +- 30 files changed, 134 insertions(+), 161 deletions(-) diff --git a/pkg/core/xds/types.go b/pkg/core/xds/types.go index 57ad8fc4c68f..1d183585a1fc 100644 --- a/pkg/core/xds/types.go +++ b/pkg/core/xds/types.go @@ -20,13 +20,30 @@ import ( type StreamID = int64 type ProxyId struct { - Mesh string - Name string - ProxyType mesh_proto.DpType + mesh string + name string + proxyType mesh_proto.DpType } -func (id ProxyId) String() string { - return fmt.Sprintf("%s.%s:%s", id.Mesh, id.Name, id.ProxyType) +func (id *ProxyId) String() string { + if id.proxyType == "" { + return fmt.Sprintf("%s.%s", id.mesh, id.name) + } + return fmt.Sprintf("%s.%s:%s", id.mesh, id.name, id.proxyType) +} + +func (id *ProxyId) ToResourceKey() core_model.ResourceKey { + return core_model.ResourceKey{ + Name: id.name, + Mesh: id.mesh, + } +} + +func (id *ProxyId) Type() mesh_proto.DpType { + if id.proxyType == "" { + return mesh_proto.RegularDpType + } + return id.proxyType } // ServiceName is a convenience type alias to clarify the meaning of string value. @@ -204,9 +221,9 @@ func (l EndpointList) Filter(selector mesh_proto.TagSelector) EndpointList { func BuildProxyId(mesh, name string, proxyType mesh_proto.DpType) *ProxyId { return &ProxyId{ - Name: name, - Mesh: mesh, - ProxyType: proxyType, + name: name, + mesh: mesh, + proxyType: proxyType, } } @@ -220,7 +237,6 @@ func ParseProxyIdFromString(id string) (*ProxyId, error) { if err != nil { return nil, err } - proxyId.ProxyType = mesh_proto.RegularDpType return proxyId, nil } proxyType := mesh_proto.DpType(parts[1]) @@ -233,9 +249,9 @@ func ParseProxyIdFromString(id string) (*ProxyId, error) { return nil, errors.New("mesh must not be empty") } return &ProxyId{ - Mesh: mesh, - Name: name, - ProxyType: proxyType, + mesh: mesh, + name: name, + proxyType: proxyType, }, nil } @@ -256,22 +272,15 @@ func ParseProxyIdFromStringOldFormat(id string) (*ProxyId, error) { return nil, errors.New("name must not be empty") } return &ProxyId{ - Mesh: mesh, - Name: name, + mesh: mesh, + name: name, }, nil } -func (id *ProxyId) ToResourceKey() core_model.ResourceKey { - return core_model.ResourceKey{ - Name: id.Name, - Mesh: id.Mesh, - } -} - func FromResourceKey(proxyType mesh_proto.DpType, key core_model.ResourceKey) ProxyId { return ProxyId{ - Mesh: key.Mesh, - Name: key.Name, - ProxyType: proxyType, + mesh: key.Mesh, + name: key.Name, + proxyType: proxyType, } } diff --git a/pkg/core/xds/types_test.go b/pkg/core/xds/types_test.go index 8ff093bd06dc..59f6d78918a8 100644 --- a/pkg/core/xds/types_test.go +++ b/pkg/core/xds/types_test.go @@ -30,28 +30,20 @@ var _ = Describe("xDS", func() { Expect(*proxyId).To(Equal(given.expected)) }, Entry("mesh and name without namespace", testCase{ - nodeID: "demo.example", - expected: core_xds.ProxyId{ - Mesh: "demo", Name: "example", ProxyType: mesh_proto.RegularDpType, - }, + nodeID: "demo.example", + expected: *core_xds.BuildProxyId("demo", "example", ""), }), Entry("name with namespace and mesh", testCase{ - nodeID: "demo.example.sample", - expected: core_xds.ProxyId{ - Mesh: "demo", Name: "example.sample", ProxyType: mesh_proto.RegularDpType, - }, + nodeID: "demo.example.sample", + expected: *core_xds.BuildProxyId("demo", "example.sample", ""), }), Entry("mesh and name without namespace and proxy type", testCase{ - nodeID: "demo.example:ingress", - expected: core_xds.ProxyId{ - Mesh: "demo", Name: "example", ProxyType: mesh_proto.IngressDpType, - }, + nodeID: "demo.example:ingress", + expected: *core_xds.BuildProxyId("demo", "example", mesh_proto.IngressDpType), }), Entry("name with namespace and mesh and proxy type", testCase{ - nodeID: "demo.example.sample:ingress", - expected: core_xds.ProxyId{ - Mesh: "demo", Name: "example.sample", ProxyType: mesh_proto.IngressDpType, - }, + nodeID: "demo.example.sample:ingress", + expected: *core_xds.BuildProxyId("demo", "example.sample", mesh_proto.IngressDpType), }), ) }) @@ -91,10 +83,7 @@ var _ = Describe("xDS", func() { Describe("ProxyId(...).ToResourceKey()", func() { It("should convert proxy ID to resource key", func() { // given - id := core_xds.ProxyId{ - Mesh: "default", - Name: "demo", - } + id := *core_xds.BuildProxyId("default", "demo", "") // when key := id.ToResourceKey() diff --git a/pkg/hds/authn/callbacks.go b/pkg/hds/authn/callbacks.go index a9b3a57a3261..ef61084aca39 100644 --- a/pkg/hds/authn/callbacks.go +++ b/pkg/hds/authn/callbacks.go @@ -148,7 +148,7 @@ func (a *authn) authenticate(credential xds_auth.Credential, nodeID string) erro backoff, _ := retry.NewConstant(a.dpNotFoundRetry.Backoff) backoff = retry.WithMaxRetries(uint64(a.dpNotFoundRetry.MaxTimes), backoff) err = retry.Do(context.Background(), backoff, func(ctx context.Context) error { - err := a.resManager.Get(context.Background(), dataplane, core_store.GetByKey(proxyId.Name, proxyId.Mesh)) + err := a.resManager.Get(context.Background(), dataplane, core_store.GetBy(proxyId.ToResourceKey())) if core_store.IsResourceNotFound(err) { return retry.RetryableError(errors.New("dataplane not found. Create Dataplane in Kuma CP first or pass it as an argument to kuma-dp")) } diff --git a/pkg/hds/tracker/callbacks.go b/pkg/hds/tracker/callbacks.go index 5097375457ca..43570a19545c 100644 --- a/pkg/hds/tracker/callbacks.go +++ b/pkg/hds/tracker/callbacks.go @@ -99,13 +99,13 @@ func (t *tracker) OnStreamClosed(streamID xds.StreamID) { func (t *tracker) OnHealthCheckRequest(streamID xds.StreamID, req *envoy_service_health.HealthCheckRequest) error { t.metrics.RequestsReceivedMetric.Inc() - id, err := xds.ParseProxyIdFromString(req.GetNode().GetId()) + proxyId, err := xds.ParseProxyIdFromString(req.GetNode().GetId()) if err != nil { t.log.Error(err, "failed to parse Dataplane Id out of HealthCheckRequest", "streamid", streamID, "req", req) return nil } - dataplaneKey := core_model.ResourceKey{Mesh: id.Mesh, Name: id.Name} + dataplaneKey := proxyId.ToResourceKey() t.Lock() defer t.Unlock() @@ -123,7 +123,7 @@ func (t *tracker) OnHealthCheckRequest(streamID xds.StreamID, req *envoy_service } // kick off watchdog for that Dataplane go t.newWatchdog(req.Node).Start(stopCh) - t.log.V(1).Info("started Watchdog for a Dataplane", "streamid", streamID, "proxyId", id, "dataplaneKey", dataplaneKey) + t.log.V(1).Info("started Watchdog for a Dataplane", "streamid", streamID, "proxyId", proxyId, "dataplaneKey", dataplaneKey) } t.dpStreams[dataplaneKey] = streams t.streamsAssociation[streamID] = dataplaneKey diff --git a/pkg/hds/tracker/healthcheck_generator.go b/pkg/hds/tracker/healthcheck_generator.go index 24c04893b1fc..cdcbddcd739e 100644 --- a/pkg/hds/tracker/healthcheck_generator.go +++ b/pkg/hds/tracker/healthcheck_generator.go @@ -12,7 +12,6 @@ import ( dp_server "github.com/kumahq/kuma/pkg/config/dp-server" "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" - "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/resources/store" "github.com/kumahq/kuma/pkg/core/xds" "github.com/kumahq/kuma/pkg/hds/cache" @@ -33,13 +32,12 @@ func NewSnapshotGenerator(readOnlyResourceManager manager.ReadOnlyResourceManage } func (g *SnapshotGenerator) GenerateSnapshot(node *envoy_core.Node) (util_xds_v3.Snapshot, error) { - proxyID, err := xds.ParseProxyIdFromString(node.Id) + proxyId, err := xds.ParseProxyIdFromString(node.Id) if err != nil { return nil, err } - dpKey := model.ResourceKey{Mesh: proxyID.Mesh, Name: proxyID.Name} dp := mesh.NewDataplaneResource() - if err := g.readOnlyResourceManager.Get(context.Background(), dp, store.GetBy(dpKey)); err != nil { + if err := g.readOnlyResourceManager.Get(context.Background(), dp, store.GetBy(proxyId.ToResourceKey())); err != nil { return nil, err } diff --git a/pkg/sds/server/v3/reconciler.go b/pkg/sds/server/v3/reconciler.go index 2e9d55387b37..f2e42dc0dcbc 100644 --- a/pkg/sds/server/v3/reconciler.go +++ b/pkg/sds/server/v3/reconciler.go @@ -59,14 +59,12 @@ type snapshotInfo struct { generation time.Time } -func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey, proxyType mesh_proto.DpType) error { - proxyID := core_xds.FromResourceKey(proxyType, dataplaneId).String() - +func (d *DataplaneReconciler) Reconcile(proxyId *core_xds.ProxyId) error { dataplane := mesh_core.NewDataplaneResource() - if err := d.readOnlyResManager.Get(context.Background(), dataplane, core_store.GetBy(dataplaneId)); err != nil { + if err := d.readOnlyResManager.Get(context.Background(), dataplane, core_store.GetBy(proxyId.ToResourceKey())); err != nil { if core_store.IsResourceNotFound(err) { - sdsServerLog.V(1).Info("Dataplane not found. Clearing the Snapshot.", "dataplaneId", dataplaneId) - if err := d.Cleanup(dataplaneId, proxyType); err != nil { + sdsServerLog.V(1).Info("Dataplane not found. Clearing the Snapshot.", "dataplaneId", proxyId.ToResourceKey()) + if err := d.Cleanup(proxyId); err != nil { return errors.Wrap(err, "could not cleanup snapshot") } return nil @@ -80,46 +78,45 @@ func (d *DataplaneReconciler) Reconcile(dataplaneId core_model.ResourceKey, prox } if !mesh.MTLSEnabled() { - sdsServerLog.V(1).Info("mTLS for Mesh disabled. Clearing the Snapshot.", "dataplaneId", dataplaneId) - if err := d.Cleanup(dataplaneId, proxyType); err != nil { + sdsServerLog.V(1).Info("mTLS for Mesh disabled. Clearing the Snapshot.", "dataplaneId", proxyId.ToResourceKey()) + if err := d.Cleanup(proxyId); err != nil { return errors.Wrap(err, "could not cleanup snapshot") } return nil } - generateSnapshot, reason, err := d.shouldGenerateSnapshot(proxyID, mesh, dataplane) + generateSnapshot, reason, err := d.shouldGenerateSnapshot(proxyId.String(), mesh, dataplane) if err != nil { return err } if generateSnapshot { - sdsServerLog.Info("Generating the Snapshot.", "dataplaneId", dataplaneId, "reason", reason) + sdsServerLog.Info("Generating the Snapshot.", "dataplaneId", proxyId.ToResourceKey(), "reason", reason) snapshot, info, err := d.generateSnapshot(dataplane, mesh) if err != nil { return err } d.sdsMetrics.CertGenerations(envoy_common.APIV3).Inc() - if err := d.cache.SetSnapshot(proxyID, snapshot); err != nil { + if err := d.cache.SetSnapshot(proxyId.String(), snapshot); err != nil { return err } - d.setSnapshotInfo(proxyID, info) + d.setSnapshotInfo(proxyId.String(), info) - if err := d.updateInsights(dataplaneId, info); err != nil { + if err := d.updateInsights(proxyId.ToResourceKey(), info); err != nil { // do not stop updating Envoy even if insights update fails - sdsServerLog.Error(err, "Could not update Dataplane Insights", "dataplaneId", dataplaneId) + sdsServerLog.Error(err, "Could not update Dataplane Insights", "dataplaneId", proxyId.ToResourceKey()) } } return nil } -func (d *DataplaneReconciler) Cleanup(dataplaneId core_model.ResourceKey, proxyType mesh_proto.DpType) error { - proxyID := core_xds.FromResourceKey(proxyType, dataplaneId).String() - if err := d.cache.SetSnapshot(proxyID, envoy_cache.Snapshot{}); err != nil { +func (d *DataplaneReconciler) Cleanup(proxyId *core_xds.ProxyId) error { + if err := d.cache.SetSnapshot(proxyId.String(), envoy_cache.Snapshot{}); err != nil { return err } d.Lock() - delete(d.proxySnapshotInfo, proxyID) + delete(d.proxySnapshotInfo, proxyId.String()) d.Unlock() return nil } diff --git a/pkg/sds/server/v3/server.go b/pkg/sds/server/v3/server.go index 1685185838e0..cfa7c9c44ed3 100644 --- a/pkg/sds/server/v3/server.go +++ b/pkg/sds/server/v3/server.go @@ -10,10 +10,7 @@ import ( envoy_server "github.com/envoyproxy/go-control-plane/pkg/server/v3" "github.com/go-logr/logr" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - "github.com/kumahq/kuma/pkg/core" - core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_runtime "github.com/kumahq/kuma/pkg/core/runtime" core_xds "github.com/kumahq/kuma/pkg/core/xds" sds_ca "github.com/kumahq/kuma/pkg/sds/ca" @@ -75,7 +72,7 @@ func RegisterSDS(rt core_runtime.Runtime, sdsMetrics *sds_metrics.Metrics) error } func syncTracker(reconciler *DataplaneReconciler, refresh time.Duration, sdsMetrics *sds_metrics.Metrics) (util_xds.Callbacks, error) { - return xds_callbacks.NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { + return xds_callbacks.NewDataplaneSyncTracker(func(proxyId *core_xds.ProxyId, streamId int64) util_watchdog.Watchdog { return &util_watchdog.SimpleWatchdog{ NewTicker: func() *time.Ticker { sdsMetrics.IncrementActiveWatchdogs(envoy_common.APIV3) @@ -86,15 +83,15 @@ func syncTracker(reconciler *DataplaneReconciler, refresh time.Duration, sdsMetr defer func() { sdsMetrics.SdsGeneration(envoy_common.APIV3).Observe(float64(core.Now().Sub(start).Milliseconds())) }() - return reconciler.Reconcile(dataplaneId, proxyType) + return reconciler.Reconcile(proxyId) }, OnError: func(err error) { sdsMetrics.SdsGenerationsErrors(envoy_common.APIV3).Inc() sdsServerLog.Error(err, "OnTick() failed") }, OnStop: func() { - if err := reconciler.Cleanup(dataplaneId, proxyType); err != nil { - sdsServerLog.Error(err, "could not cleanup sync", "dataplane", dataplaneId) + if err := reconciler.Cleanup(proxyId); err != nil { + sdsServerLog.Error(err, "could not cleanup sync", "dataplane", proxyId.ToResourceKey()) } sdsMetrics.DecrementActiveWatchdogs(envoy_common.APIV3) }, diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index 40436b92f434..0927511ee7e3 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -10,8 +10,6 @@ import ( "google.golang.org/grpc/metadata" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - core_model "github.com/kumahq/kuma/pkg/core/resources/model" - core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_manager "github.com/kumahq/kuma/pkg/core/resources/manager" core_store "github.com/kumahq/kuma/pkg/core/resources/store" @@ -124,7 +122,7 @@ func (a *authCallbacks) authenticate(credential Credential, req util_xds.Discove if err != nil { return errors.Wrap(err, "request must have a valid Proxy Id") } - switch proxyId.ProxyType { + switch proxyId.Type() { case mesh_proto.IngressDpType: return a.authenticateZoneIngress(credential, req) default: @@ -145,7 +143,7 @@ func (a *authCallbacks) authenticateZoneIngress(credential Credential, req util_ backoff, _ := retry.NewConstant(a.dpNotFoundRetry.Backoff) backoff = retry.WithMaxRetries(uint64(a.dpNotFoundRetry.MaxTimes), backoff) err = retry.Do(context.Background(), backoff, func(ctx context.Context) error { - err := a.resManager.Get(ctx, zoneIngress, core_store.GetByKey(proxyId.Name, core_model.NoMesh)) + err := a.resManager.Get(ctx, zoneIngress, core_store.GetBy(proxyId.ToResourceKey())) if core_store.IsResourceNotFound(err) { return retry.RetryableError(errors.New("zoneIngress not found. Create ZoneIngress in Kuma CP first or pass it as an argument to kuma-dp")) } @@ -175,7 +173,7 @@ func (a *authCallbacks) authenticateRegularDataplane(credential Credential, req backoff, _ := retry.NewConstant(a.dpNotFoundRetry.Backoff) backoff = retry.WithMaxRetries(uint64(a.dpNotFoundRetry.MaxTimes), backoff) err = retry.Do(context.Background(), backoff, func(ctx context.Context) error { - err := a.resManager.Get(ctx, dataplane, core_store.GetByKey(proxyId.Name, proxyId.Mesh)) + err := a.resManager.Get(ctx, dataplane, core_store.GetBy(proxyId.ToResourceKey())) if core_store.IsResourceNotFound(err) { return retry.RetryableError(errors.New("dataplane not found. Create Dataplane in Kuma CP first or pass it as an argument to kuma-dp")) } diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index dee7bd57a217..4f8cd93ac08e 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -208,7 +208,7 @@ func (b *bootstrapGenerator) zoneIngressFor(ctx context.Context, request types.B return zoneIngress, nil } else { zoneIngress := core_mesh.NewZoneIngressResource() - if err := b.resManager.Get(ctx, zoneIngress, core_store.GetByKey(proxyId.Name, core_model.NoMesh)); err != nil { + if err := b.resManager.Get(ctx, zoneIngress, core_store.GetBy(proxyId.ToResourceKey())); err != nil { return nil, err } return zoneIngress, nil diff --git a/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go b/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go index ac987b3e83dd..dde7cbab85b4 100644 --- a/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go +++ b/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go @@ -10,7 +10,6 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" - "github.com/kumahq/kuma/pkg/core/xds" core_xds "github.com/kumahq/kuma/pkg/core/xds" util_proto "github.com/kumahq/kuma/pkg/util/proto" ) @@ -35,10 +34,7 @@ var _ = Describe("HttpAccessLogConfigurer", func() { sourceService := "web" destinationService := "backend" proxy := &core_xds.Proxy{ - Id: xds.ProxyId{ - Name: "web", - Mesh: "example", - }, + Id: *core_xds.BuildProxyId("web", "example", ""), Dataplane: &mesh_core.DataplaneResource{ Spec: &mesh_proto.Dataplane{ Networking: &mesh_proto.Dataplane_Networking{ diff --git a/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go b/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go index 672604ec5213..e00a8ff10c16 100644 --- a/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go +++ b/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go @@ -9,7 +9,6 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" - "github.com/kumahq/kuma/pkg/core/xds" core_xds "github.com/kumahq/kuma/pkg/core/xds" util_proto "github.com/kumahq/kuma/pkg/util/proto" envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" @@ -35,10 +34,7 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { sourceService := "backend" destinationService := "db" proxy := &core_xds.Proxy{ - Id: xds.ProxyId{ - Name: "backend", - Mesh: "example", - }, + Id: *core_xds.BuildProxyId("example", "backend", ""), Dataplane: &mesh_core.DataplaneResource{ Spec: &mesh_proto.Dataplane{ Networking: &mesh_proto.Dataplane_Networking{ diff --git a/pkg/xds/generator/dns_generator_test.go b/pkg/xds/generator/dns_generator_test.go index 908d33f7c45b..598ee9f83a28 100644 --- a/pkg/xds/generator/dns_generator_test.go +++ b/pkg/xds/generator/dns_generator_test.go @@ -70,7 +70,7 @@ var _ = Describe("DNSGenerator", func() { Expect(err).ToNot(HaveOccurred()) Expect(util_proto.FromYAML(dpBytes, &dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "1", diff --git a/pkg/xds/generator/inbound_proxy_generator_test.go b/pkg/xds/generator/inbound_proxy_generator_test.go index 560bee2e2bb5..a89475ec41ed 100644 --- a/pkg/xds/generator/inbound_proxy_generator_test.go +++ b/pkg/xds/generator/inbound_proxy_generator_test.go @@ -64,7 +64,7 @@ var _ = Describe("InboundProxyGenerator", func() { Expect(err).ToNot(HaveOccurred()) Expect(util_proto.FromYAML(dpBytes, &dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "1", diff --git a/pkg/xds/generator/ingress_generator_test.go b/pkg/xds/generator/ingress_generator_test.go index 72b67d72640a..791dd8f5d077 100644 --- a/pkg/xds/generator/ingress_generator_test.go +++ b/pkg/xds/generator/ingress_generator_test.go @@ -43,7 +43,7 @@ var _ = Describe("IngressGenerator", func() { }) Expect(err).ToNot(HaveOccurred()) proxy := &core_xds.Proxy{ - Id: core_xds.ProxyId{Name: "ingress", Mesh: "default"}, + Id: *core_xds.BuildProxyId("default", "ingress", ""), ZoneIngress: zoneIngress, APIVersion: envoy_common.APIV3, Routing: core_xds.Routing{ diff --git a/pkg/xds/generator/outbound_proxy_generator_test.go b/pkg/xds/generator/outbound_proxy_generator_test.go index fb5778c876f4..638a68e08b39 100644 --- a/pkg/xds/generator/outbound_proxy_generator_test.go +++ b/pkg/xds/generator/outbound_proxy_generator_test.go @@ -159,7 +159,7 @@ var _ = Describe("OutboundProxyGenerator", func() { }, } proxy := &model.Proxy{ - Id: model.ProxyId{Name: "side-car", Mesh: "default"}, + Id: *model.BuildProxyId("default", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "dp-1", @@ -500,7 +500,7 @@ var _ = Describe("OutboundProxyGenerator", func() { }, } proxy := &model.Proxy{ - Id: model.ProxyId{Name: "side-car", Mesh: "default"}, + Id: *model.BuildProxyId("default", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "1", diff --git a/pkg/xds/generator/prometheus_endpoint_generator_test.go b/pkg/xds/generator/prometheus_endpoint_generator_test.go index fef5a4d5fff9..e1d22b7eab77 100644 --- a/pkg/xds/generator/prometheus_endpoint_generator_test.go +++ b/pkg/xds/generator/prometheus_endpoint_generator_test.go @@ -52,7 +52,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", @@ -75,7 +75,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -121,7 +121,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -168,7 +168,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -243,7 +243,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -288,7 +288,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -360,7 +360,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -423,7 +423,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -487,7 +487,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -542,7 +542,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, } proxy := &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ diff --git a/pkg/xds/generator/proxy_template_profile_source_test.go b/pkg/xds/generator/proxy_template_profile_source_test.go index 79bc881c4df8..467c71f50b03 100644 --- a/pkg/xds/generator/proxy_template_profile_source_test.go +++ b/pkg/xds/generator/proxy_template_profile_source_test.go @@ -99,7 +99,7 @@ var _ = Describe("ProxyTemplateProfileSource", func() { Expect(util_proto.FromYAML([]byte(given.dataplane), dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", diff --git a/pkg/xds/generator/proxy_template_raw_source_test.go b/pkg/xds/generator/proxy_template_raw_source_test.go index 1515139b2365..b862e9fc2f0f 100644 --- a/pkg/xds/generator/proxy_template_raw_source_test.go +++ b/pkg/xds/generator/proxy_template_raw_source_test.go @@ -43,7 +43,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }, Entry("should fail when `resource` field is empty", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -72,7 +72,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field is neither a YAML nor a JSON", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -100,7 +100,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field has unknown @type", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -130,7 +130,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field is a YAML without '@type' field", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -171,7 +171,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field is an invalid xDS resource", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -247,7 +247,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }, Entry("should support empty resource list", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -271,7 +271,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should support Listener resource as YAML", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -332,7 +332,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should support Cluster resource as YAML", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -391,7 +391,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should support Cluster resource as JSON", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", diff --git a/pkg/xds/generator/proxy_template_test.go b/pkg/xds/generator/proxy_template_test.go index 07942a693e3e..0402e29d58cf 100644 --- a/pkg/xds/generator/proxy_template_test.go +++ b/pkg/xds/generator/proxy_template_test.go @@ -60,7 +60,7 @@ var _ = Describe("ProxyTemplateGenerator", func() { }, Entry("should fail when raw xDS resource is not valid", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", @@ -152,7 +152,7 @@ var _ = Describe("ProxyTemplateGenerator", func() { dataplane := &mesh_proto.Dataplane{} Expect(util_proto.FromYAML([]byte(given.dataplane), dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", diff --git a/pkg/xds/generator/tracing_generator_test.go b/pkg/xds/generator/tracing_generator_test.go index c083fc035721..9e158e831485 100644 --- a/pkg/xds/generator/tracing_generator_test.go +++ b/pkg/xds/generator/tracing_generator_test.go @@ -41,7 +41,7 @@ var _ = Describe("TracingProxyGenerator", func() { }, Entry("Mesh has no Tracing configuration", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", @@ -74,7 +74,7 @@ var _ = Describe("TracingProxyGenerator", func() { }, Entry("should create cluster for Zipkin", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "demo.backend-01"}, + Id: *model.BuildProxyId("", "demo.backend-01", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", diff --git a/pkg/xds/generator/transparent_proxy_generator_test.go b/pkg/xds/generator/transparent_proxy_generator_test.go index 1c3d7d314f20..613b0d3aa22c 100644 --- a/pkg/xds/generator/transparent_proxy_generator_test.go +++ b/pkg/xds/generator/transparent_proxy_generator_test.go @@ -57,7 +57,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }, Entry("transparent_proxying=false", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -69,7 +69,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }), Entry("transparent_proxying=true", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -100,7 +100,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }), Entry("transparent_proxying=true with logs", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -131,7 +131,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }), Entry("transparent_proxying=true ipv6", testCase{ proxy: &model.Proxy{ - Id: model.ProxyId{Name: "side-car"}, + Id: *model.BuildProxyId("", "side-car", ""), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", diff --git a/pkg/xds/server/callbacks/dataplane_lifecycle.go b/pkg/xds/server/callbacks/dataplane_lifecycle.go index 1963c5f8dbec..5ce14e483d17 100644 --- a/pkg/xds/server/callbacks/dataplane_lifecycle.go +++ b/pkg/xds/server/callbacks/dataplane_lifecycle.go @@ -108,7 +108,7 @@ func (d *DataplaneLifecycle) OnStreamRequest(streamID int64, request util_xds.Di return nil } - if proxyId.ProxyType == mesh_proto.RegularDpType && md.GetDataplaneResource() != nil { + if proxyId.Type() == mesh_proto.RegularDpType && md.GetDataplaneResource() != nil { lifecycleLog.Info("registering dataplane", "dataplane", md.GetDataplaneResource(), "streamID", streamID, "nodeID", request.NodeId()) if err := d.registerDataplane(md.GetDataplaneResource()); err != nil { return errors.Wrap(err, "could not register dataplane passed in kuma-dp run") @@ -119,7 +119,7 @@ func (d *DataplaneLifecycle) OnStreamRequest(streamID int64, request util_xds.Di return nil } - if proxyId.ProxyType == mesh_proto.IngressDpType && md.ZoneIngressResource != nil { + if proxyId.Type() == mesh_proto.IngressDpType && md.ZoneIngressResource != nil { lifecycleLog.Info("registering zone ingress", "zoneIngress", md.ZoneIngressResource, "streamID", streamID, "nodeID", request.NodeId()) if err := d.registerZoneIngress(md.ZoneIngressResource); err != nil { return errors.Wrap(err, "could not register zone ingress passed in kuma-dp run") diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index 5d0539b32fef..3a7009521e69 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -116,13 +116,13 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di // infer Dataplane id if state.dataplaneId == (core_model.ResourceKey{}) { - if id, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { - state.dataplaneId = core_model.ResourceKey{Mesh: id.Mesh, Name: id.Name} + if proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { + state.dataplaneId = proxyId.ToResourceKey() if err := readVersion(req.Metadata(), state.subscription.Version); err != nil { statusTrackerLog.Error(err, "failed to extract version out of the Envoy metadata", "streamid", streamID, "metadata", req.Metadata()) } // kick off async Dataplane status flusher - if id.ProxyType != mesh_proto.IngressDpType { + if proxyId.Type() != mesh_proto.IngressDpType { go c.createStatusSink(state).Start(state.stop) } } else { diff --git a/pkg/xds/server/callbacks/dataplane_sync_tracker.go b/pkg/xds/server/callbacks/dataplane_sync_tracker.go index ea6cd1e200df..5de6ae83ad18 100644 --- a/pkg/xds/server/callbacks/dataplane_sync_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_sync_tracker.go @@ -4,7 +4,6 @@ import ( "context" stdsync "sync" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_xds "github.com/kumahq/kuma/pkg/core/xds" @@ -16,7 +15,7 @@ var ( dataplaneSyncTrackerLog = core.Log.WithName("xds-server").WithName("dataplane-sync-tracker") ) -type NewDataplaneWatchdogFunc func(dataplaneId core_model.ResourceKey, streamId core_xds.StreamID, proxyType mesh_proto.DpType) util_watchdog.Watchdog +type NewDataplaneWatchdogFunc func(proxyId *core_xds.ProxyId, streamId core_xds.StreamID) util_watchdog.Watchdog func NewDataplaneSyncTracker(factoryFunc NewDataplaneWatchdogFunc) util_xds.Callbacks { return &dataplaneSyncTracker{ @@ -84,8 +83,8 @@ func (t *dataplaneSyncTracker) OnStreamRequest(streamID core_xds.StreamID, req u return nil } - if id, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { - dataplaneKey := core_model.ResourceKey{Mesh: id.Mesh, Name: id.Name} + if proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { + dataplaneKey := proxyId.ToResourceKey() t.Lock() defer t.Unlock() streams := t.dpStreams[dataplaneKey] @@ -99,8 +98,8 @@ func (t *dataplaneSyncTracker) OnStreamRequest(streamID core_xds.StreamID, req u close(stopCh) } // kick off watchdog for that Dataplane - go t.newDataplaneWatchdog(dataplaneKey, streamID, id.ProxyType).Start(stopCh) - dataplaneSyncTrackerLog.V(1).Info("started Watchdog for a Dataplane", "streamid", streamID, "proxyId", id, "dataplaneKey", dataplaneKey) + go t.newDataplaneWatchdog(proxyId, streamID).Start(stopCh) + dataplaneSyncTrackerLog.V(1).Info("started Watchdog for a Dataplane", "streamid", streamID, "proxyId", proxyId, "dataplaneKey", dataplaneKey) } t.dpStreams[dataplaneKey] = streams t.streamsAssociation[streamID] = dataplaneKey diff --git a/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go b/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go index 005303a914cf..c179aebf91aa 100644 --- a/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go +++ b/pkg/xds/server/callbacks/dataplane_sync_tracker_test.go @@ -8,7 +8,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + core_xds "github.com/kumahq/kuma/pkg/core/xds" core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/test" @@ -81,13 +81,13 @@ var _ = Describe("Sync", func() { watchdogCh := make(chan core_model.ResourceKey) // setup - tracker := NewDataplaneSyncTracker(NewDataplaneWatchdogFunc(func(dataplaneId core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { + tracker := NewDataplaneSyncTracker(func(proxyId *core_xds.ProxyId, streamId int64) util_watchdog.Watchdog { return WatchdogFunc(func(stop <-chan struct{}) { - watchdogCh <- dataplaneId + watchdogCh <- proxyId.ToResourceKey() <-stop close(watchdogCh) }) - })) + }) callbacks := util_xds_v2.AdaptCallbacks(tracker) // given @@ -138,7 +138,7 @@ var _ = Describe("Sync", func() { It("should start only one watchdog per dataplane", func() { // setup var activeWatchdogs int32 - tracker := NewDataplaneSyncTracker(func(dataplaneId core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { + tracker := NewDataplaneSyncTracker(func(proxyId *core_xds.ProxyId, streamId int64) util_watchdog.Watchdog { return WatchdogFunc(func(stop <-chan struct{}) { atomic.AddInt32(&activeWatchdogs, 1) <-stop diff --git a/pkg/xds/server/v3/reconcile_test.go b/pkg/xds/server/v3/reconcile_test.go index 82b4f598afe3..3f07e51d963d 100644 --- a/pkg/xds/server/v3/reconcile_test.go +++ b/pkg/xds/server/v3/reconcile_test.go @@ -87,7 +87,7 @@ var _ = Describe("Reconcile", func() { }, } - It("should generate a Snaphot per Envoy Node", func() { + It("should generate a Snapshot per Envoy Node", func() { // given snapshots := make(chan envoy_cache.Snapshot, 3) snapshots <- snapshot // initial Dataplane configuration @@ -115,10 +115,7 @@ var _ = Describe("Reconcile", func() { By("simulating discovery event") // when proxy := &xds_model.Proxy{ - Id: xds_model.ProxyId{ - Mesh: "demo", - Name: "example", - }, + Id: *xds_model.BuildProxyId("demo", "example", ""), Dataplane: dataplane, } err := r.Reconcile(xds_context.Context{}, proxy) diff --git a/pkg/xds/server/v3/snapshot_generator_test.go b/pkg/xds/server/v3/snapshot_generator_test.go index 93fd2cd78586..f116f0b6fc66 100644 --- a/pkg/xds/server/v3/snapshot_generator_test.go +++ b/pkg/xds/server/v3/snapshot_generator_test.go @@ -67,7 +67,7 @@ var _ = Describe("Reconcile", func() { Expect(util_proto.FromYAML(dpBytes, &dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: model.ProxyId{Name: "demo.web1"}, + Id: *model.BuildProxyId("", "demo.web1", ""), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ diff --git a/pkg/xds/sync/dataplane_watchdog.go b/pkg/xds/sync/dataplane_watchdog.go index 5027007c1f80..92f5eeadffb8 100644 --- a/pkg/xds/sync/dataplane_watchdog.go +++ b/pkg/xds/sync/dataplane_watchdog.go @@ -12,7 +12,7 @@ import ( "github.com/kumahq/kuma/pkg/core" "github.com/kumahq/kuma/pkg/core/resources/manager" core_model "github.com/kumahq/kuma/pkg/core/resources/model" - "github.com/kumahq/kuma/pkg/core/xds" + core_xds "github.com/kumahq/kuma/pkg/core/xds" "github.com/kumahq/kuma/pkg/xds/cache/mesh" ) @@ -38,13 +38,13 @@ type DataplaneWatchdog struct { proxyTypeSettled bool } -func NewDataplaneWatchdog(deps DataplaneWatchdogDependencies, key core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) *DataplaneWatchdog { +func NewDataplaneWatchdog(deps DataplaneWatchdogDependencies, proxyId *core_xds.ProxyId, streamId int64) *DataplaneWatchdog { return &DataplaneWatchdog{ DataplaneWatchdogDependencies: deps, - key: key, + key: proxyId.ToResourceKey(), streamId: streamId, log: core.Log.WithValues("key", "key", "streamID", streamId), - dpType: proxyType, + dpType: proxyId.Type(), proxyTypeSettled: false, } } @@ -75,7 +75,7 @@ func (d *DataplaneWatchdog) Sync() error { } func (d *DataplaneWatchdog) Cleanup() error { - proxyID := xds.FromResourceKey(d.dpType, d.key) + proxyID := core_xds.FromResourceKey(d.dpType, d.key) switch d.dpType { case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: return d.dataplaneReconciler.Clear(&proxyID) diff --git a/pkg/xds/sync/dataplane_watchdog_factory.go b/pkg/xds/sync/dataplane_watchdog_factory.go index a4dfddcac71f..e7b06eada587 100644 --- a/pkg/xds/sync/dataplane_watchdog_factory.go +++ b/pkg/xds/sync/dataplane_watchdog_factory.go @@ -3,9 +3,8 @@ package sync import ( "time" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" - core_model "github.com/kumahq/kuma/pkg/core/resources/model" + core_xds "github.com/kumahq/kuma/pkg/core/xds" util_watchdog "github.com/kumahq/kuma/pkg/util/watchdog" xds_metrics "github.com/kumahq/kuma/pkg/xds/metrics" ) @@ -29,9 +28,9 @@ func NewDataplaneWatchdogFactory( }, nil } -func (d *dataplaneWatchdogFactory) New(key core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog { - log := xdsServerLog.WithName("dataplane-sync-watchdog").WithValues("dataplaneKey", key) - dataplaneWatchdog := NewDataplaneWatchdog(d.deps, key, streamId, proxyType) +func (d *dataplaneWatchdogFactory) New(proxyId *core_xds.ProxyId, streamId int64) util_watchdog.Watchdog { + log := xdsServerLog.WithName("dataplane-sync-watchdog").WithValues("dataplaneKey", proxyId.ToResourceKey()) + dataplaneWatchdog := NewDataplaneWatchdog(d.deps, proxyId, streamId) return &util_watchdog.SimpleWatchdog{ NewTicker: func() *time.Ticker { return time.NewTicker(d.refreshInterval) diff --git a/pkg/xds/sync/interfaces.go b/pkg/xds/sync/interfaces.go index 0418d560356f..576801f2c2af 100644 --- a/pkg/xds/sync/interfaces.go +++ b/pkg/xds/sync/interfaces.go @@ -1,8 +1,6 @@ package sync import ( - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - core_model "github.com/kumahq/kuma/pkg/core/resources/model" core_xds "github.com/kumahq/kuma/pkg/core/xds" util_watchdog "github.com/kumahq/kuma/pkg/util/watchdog" xds_context "github.com/kumahq/kuma/pkg/xds/context" @@ -24,5 +22,5 @@ type SnapshotReconciler interface { // DataplaneWatchdogFactory returns a Watchdog that creates a new XdsContext and Proxy and executes SnapshotReconciler if there is any change type DataplaneWatchdogFactory interface { - New(key core_model.ResourceKey, streamId int64, proxyType mesh_proto.DpType) util_watchdog.Watchdog + New(proxyId *core_xds.ProxyId, streamId int64) util_watchdog.Watchdog } From 7171363ef1efc9e6ba7f55308b55a1edbe49a71a Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Fri, 11 Jun 2021 11:56:43 +0700 Subject: [PATCH 07/19] chore(kuma-cp) fix integration tests Signed-off-by: Ilya Lobkov --- .../dataplane/envoy/remote_bootstrap_test.go | 3 ++ ...tall-control-plane.cni-enabled.golden.yaml | 27 ++++++++++++++++ ...install-control-plane.defaults.golden.yaml | 27 ++++++++++++++++ .../install-control-plane.global.golden.yaml | 27 ++++++++++++++++ ...ontrol-plane.override-env-vars.golden.yaml | 27 ++++++++++++++++ ...nstall-control-plane.overrides.golden.yaml | 27 ++++++++++++++++ ...all-control-plane.with-ingress.golden.yaml | 31 +++++++++++++++++-- .../install-control-plane.zone.golden.yaml | 27 ++++++++++++++++ .../testdata/install-crds.all.golden.yaml | 26 ++++++++++++++++ pkg/defaults/components_test.go | 1 - pkg/kds/global/components_test.go | 4 ++- pkg/kds/server/server_test.go | 2 ++ pkg/test/kds/samples/resources.go | 14 +++++++++ .../testdata/bootstrap.k8s.golden.yaml | 2 +- .../testdata/bootstrap.overridden.golden.yaml | 2 +- .../testdata/bootstrap.universal.golden.yaml | 2 +- ....custom-config-minimal-request.golden.yaml | 2 +- .../generator.custom-config.golden.yaml | 2 +- ...default-config-minimal-request.golden.yaml | 2 +- .../generator.default-config.golden.yaml | 2 +- ...ator.default-config.kubernetes.golden.yaml | 2 +- ...default-config.kubernetes.ipv6.golden.yaml | 2 +- 22 files changed, 248 insertions(+), 13 deletions(-) diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go index 5a06a3804986..99fe262ed24f 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go @@ -107,6 +107,7 @@ var _ = Describe("Remote Bootstrap", func() { { "mesh": "demo", "name": "sample", + "proxyType": "regular", "adminPort": 4321, "dataplaneToken": "token", "dataplaneResource": "{\"type\":\"Dataplane\",\"mesh\":\"demo\",\"name\":\"sample\",\"creationTime\":\"0001-01-01T00:00:00Z\",\"modificationTime\":\"0001-01-01T00:00:00Z\"}", @@ -153,6 +154,7 @@ var _ = Describe("Remote Bootstrap", func() { { "mesh": "demo", "name": "sample", + "proxyType": "regular", "adminPort": 4321, "dataplaneToken": "token", "dataplaneResource": "{\"type\":\"Dataplane\",\"mesh\":\"demo\",\"name\":\"sample\",\"creationTime\":\"0001-01-01T00:00:00Z\",\"modificationTime\":\"0001-01-01T00:00:00Z\"}", @@ -195,6 +197,7 @@ var _ = Describe("Remote Bootstrap", func() { { "mesh": "demo", "name": "sample", + "proxyType": "regular", "dataplaneToken": "token", "dataplaneResource": "{\"type\":\"Dataplane\",\"mesh\":\"demo\",\"name\":\"sample\",\"creationTime\":\"0001-01-01T00:00:00Z\",\"modificationTime\":\"0001-01-01T00:00:00Z\"}", "version": { diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml index 66097c1c240a..cdf8c12f1d4b 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml @@ -328,6 +328,32 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: creationTimestamp: null name: dataplaneinsights.kuma.io @@ -593,6 +619,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml index 1702ad7a2f46..6144caa627d9 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml @@ -292,6 +292,32 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: creationTimestamp: null name: dataplaneinsights.kuma.io @@ -540,6 +566,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml index e32d1e4461e1..7744ebb3b32a 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml @@ -292,6 +292,32 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: creationTimestamp: null name: dataplaneinsights.kuma.io @@ -540,6 +566,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml index 0f9a562f4451..85c0911e6898 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml @@ -292,6 +292,32 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: creationTimestamp: null name: dataplaneinsights.kuma.io @@ -540,6 +566,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml index 42b5ffb2d81f..ae2dfedf678b 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml @@ -292,6 +292,32 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: creationTimestamp: null name: dataplaneinsights.kuma.io @@ -540,6 +566,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml index 8d2a46bd9cc6..0edcf5e25258 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml @@ -301,6 +301,32 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: creationTimestamp: null name: dataplaneinsights.kuma.io @@ -549,6 +575,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates @@ -882,8 +909,6 @@ spec: value: "https://kuma-control-plane.kuma-system:5678" - name: KUMA_CONTROL_PLANE_CA_CERT_FILE value: /var/run/secrets/kuma.io/tls-cert/ca.crt - - name: KUMA_DATAPLANE_MESH - value: default - name: KUMA_DATAPLANE_NAME value: $(POD_NAME).$(POD_NAMESPACE) - name: KUMA_DATAPLANE_ADMIN_PORT @@ -892,6 +917,8 @@ spec: value: 60s - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH value: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: KUMA_DATAPLANE_IS_INGRESS + value: "true" args: - run - --log-level=info diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml index 8d34c41d37f5..ab08f0825bd4 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml @@ -292,6 +292,32 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: creationTimestamp: null name: dataplaneinsights.kuma.io @@ -540,6 +566,7 @@ rules: - meshes - zones - zoneinsights + - zoneingresses - meshinsights - serviceinsights - proxytemplates diff --git a/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml b/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml index c0378c0aad49..c15499f72b2e 100644 --- a/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-crds.all.golden.yaml @@ -457,6 +457,32 @@ spec: schema: openAPIV3Schema: description: Zone is the Schema for the zone API + properties: + mesh: + type: string + spec: + x-kubernetes-preserve-unknown-fields: true + type: object + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: zoneingresses.kuma.io +spec: + group: kuma.io + names: + kind: ZoneIngress + plural: zoneingresses + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: ZoneIngress is the Schema for the zone ingress API properties: mesh: type: string diff --git a/pkg/defaults/components_test.go b/pkg/defaults/components_test.go index 07c5b654fd82..39df61d69e72 100644 --- a/pkg/defaults/components_test.go +++ b/pkg/defaults/components_test.go @@ -153,5 +153,4 @@ var _ = Describe("Defaults Component", func() { Expect(actual.Spec.GetData().GetValue()).To(Equal([]byte("hello"))) }) }) - }) diff --git a/pkg/kds/global/components_test.go b/pkg/kds/global/components_test.go index 85ce7fff416a..041fe10eeebe 100644 --- a/pkg/kds/global/components_test.go +++ b/pkg/kds/global/components_test.go @@ -190,10 +190,12 @@ var _ = Describe("Global Sync", func() { } } - // plus 2 global-scope types + // plus 4 global-scope types extraTypes := []model.ResourceType{ mesh.MeshType, + mesh.ZoneIngressType, system.ConfigType, + system.GlobalSecretType, } actualProvidedTypes = append(actualProvidedTypes, extraTypes...) diff --git a/pkg/kds/server/server_test.go b/pkg/kds/server/server_test.go index 37e27364e5d4..534684db8dba 100644 --- a/pkg/kds/server/server_test.go +++ b/pkg/kds/server/server_test.go @@ -65,6 +65,7 @@ var _ = Describe("KDS Server", func() { Expect([]proto.Message{ kds_samples.CircuitBreaker, kds_samples.Ingress, // mesh.DataplaneType + kds_samples.ZoneIngress, kds_samples.DataplaneInsight, kds_samples.ExternalService, kds_samples.FaultInjection, @@ -78,6 +79,7 @@ var _ = Describe("KDS Server", func() { kds_samples.TrafficRoute, kds_samples.TrafficTrace, kds_samples.Secret, + kds_samples.GlobalSecret, kds_samples.Config, }). To(HaveLen(len(kds.SupportedTypes))) diff --git a/pkg/test/kds/samples/resources.go b/pkg/test/kds/samples/resources.go index a79c4ecc84f5..8a220e3b193e 100644 --- a/pkg/test/kds/samples/resources.go +++ b/pkg/test/kds/samples/resources.go @@ -97,6 +97,17 @@ var ( Address: "192.168.0.1", }, } + ZoneIngress = &mesh_proto.ZoneIngress{ + Address: "127.0.0.1", + Port: 80, + AdvertisedAddress: "192.168.0.1", + AdvertisedPort: 10001, + AvailableServices: []*mesh_proto.ZoneIngress_AvailableService{{ + Tags: map[string]string{ + "service": "backend", + }}, + }, + } ExternalService = &mesh_proto.ExternalService{ Networking: &mesh_proto.ExternalService_Networking{ Address: "192.168.0.1", @@ -275,6 +286,9 @@ var ( Secret = &system_proto.Secret{ Data: &wrappers.BytesValue{Value: []byte("secret key")}, } + GlobalSecret = &system_proto.Secret{ + Data: &wrappers.BytesValue{Value: []byte("global secret key")}, + } Config = &system_proto.Config{ Config: "sample config", } diff --git a/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml b/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml index 4c8916154c6b..289290d2a5ad 100644 --- a/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml +++ b/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml @@ -53,7 +53,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: default.dp-1.default + id: default.dp-1.default:regular metadata: dataplaneTokenPath: /tmp/token version: diff --git a/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml b/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml index 2bdce08e32e5..6282427d4a5d 100644 --- a/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml +++ b/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml @@ -59,7 +59,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: default.dp-1.default + id: default.dp-1.default:regular metadata: dataplane.admin.port: "1234" dataplaneTokenPath: /tmp/token diff --git a/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml b/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml index 14b3eaddc870..54fb0bb9455b 100644 --- a/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml +++ b/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml @@ -53,7 +53,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: default.dp-1 + id: default.dp-1:regular metadata: dataplaneTokenPath: /tmp/token version: diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml index ceee99e20b10..5d436934e9e6 100644 --- a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml @@ -33,7 +33,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace + id: mesh.name.namespace:regular metadata: dataplane.admin.port: "9902" version: diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml index 6fd8d99b4578..597245208273 100644 --- a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml @@ -59,7 +59,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace + id: mesh.name.namespace:regular metadata: dataplane.admin.port: "1234" dataplane.resource: ' { "type": "Dataplane", "mesh": "mesh", "name": "name.namespace", diff --git a/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml index 6324d5b8f4bc..0585aeb85f9b 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml @@ -27,7 +27,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace + id: mesh.name.namespace:regular metadata: version: envoy: diff --git a/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml index 746680dc8fd6..d4734834500a 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml @@ -39,7 +39,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace + id: mesh.name.namespace:regular metadata: dataplane.admin.port: "1234" dataplane.dns.empty.port: "53002" diff --git a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml index f3329800641c..86961947b297 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml @@ -39,7 +39,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace + id: mesh.name.namespace:regular metadata: dataplane.admin.port: "1234" dataplaneTokenPath: /tmp/token diff --git a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml index 8c9403bf9964..c157fc37ff5b 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml @@ -39,7 +39,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace + id: mesh.name.namespace:regular metadata: dataplane.admin.port: "1234" dataplaneTokenPath: /tmp/token From 9487905b2b7c6d49367c4e32ae116218e1baec3e Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Fri, 11 Jun 2021 11:58:03 +0700 Subject: [PATCH 08/19] chore(kuma-cp) make check -C api Signed-off-by: Ilya Lobkov --- api/mesh/v1alpha1/zone_ingress.pb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/mesh/v1alpha1/zone_ingress.pb.go b/api/mesh/v1alpha1/zone_ingress.pb.go index 1d32a4af3f11..10c3808845f2 100644 --- a/api/mesh/v1alpha1/zone_ingress.pb.go +++ b/api/mesh/v1alpha1/zone_ingress.pb.go @@ -46,7 +46,8 @@ type ZoneIngress struct { // AdvertisedPort defines port on which ZoneIngress is accessible to other // Kuma clusters. AdvertisedPort uint32 `protobuf:"varint,4,opt,name=advertisedPort,proto3" json:"advertisedPort,omitempty"` - // Zone field contains Zone name where ingress is serving, field will be automatically set by Global Kuma CP + // Zone field contains Zone name where ingress is serving, field will be + // automatically set by Global Kuma CP Zone string `protobuf:"bytes,6,opt,name=zone,proto3" json:"zone,omitempty"` // AvailableService contains tags that represent unique subset of // endpoints From e0a0cb4d1264deb5cfe71d1410c5e8ac10008f3a Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Fri, 11 Jun 2021 14:45:02 +0700 Subject: [PATCH 09/19] chore(kuma-cp) fix integration tests Signed-off-by: Ilya Lobkov --- pkg/kds/zone/components_test.go | 2 ++ pkg/xds/cache/cla/cache_test.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/kds/zone/components_test.go b/pkg/kds/zone/components_test.go index 15b510508189..fcdd1992847f 100644 --- a/pkg/kds/zone/components_test.go +++ b/pkg/kds/zone/components_test.go @@ -169,7 +169,9 @@ var _ = Describe("Zone Sync", func() { // plus 2 global-scope types extraTypes := []model.ResourceType{ mesh.MeshType, + mesh.ZoneIngressType, system.ConfigType, + system.GlobalSecretType, } actualConsumedTypes = append(actualConsumedTypes, extraTypes...) diff --git a/pkg/xds/cache/cla/cache_test.go b/pkg/xds/cache/cla/cache_test.go index 5c14f668cfdc..9bd3871ce14f 100644 --- a/pkg/xds/cache/cla/cache_test.go +++ b/pkg/xds/cache/cla/cache_test.go @@ -97,8 +97,13 @@ var _ = Describe("ClusterLoadAssignment Cache", func() { By("getting CLA for the first time") cla, err := claCache.GetCLA(context.Background(), "mesh-0", "", envoy_common.NewCluster(envoy_common.WithService("backend")), envoy_common.APIV3) Expect(err).ToNot(HaveOccurred()) + // 1 Get request: + // - GetMesh Expect(countingManager.getQueries).To(Equal(1)) - Expect(countingManager.listQueries).To(Equal(1)) + // 2 List request: + // - GetDataplanes + // - GetZoneIngresses + Expect(countingManager.listQueries).To(Equal(2)) expected, err := ioutil.ReadFile(filepath.Join("testdata", "cla.get.0.json")) Expect(err).ToNot(HaveOccurred()) @@ -111,7 +116,7 @@ var _ = Describe("ClusterLoadAssignment Cache", func() { _, err = claCache.GetCLA(context.Background(), "mesh-0", "", envoy_common.NewCluster(envoy_common.WithService("backend")), envoy_common.APIV3) Expect(err).ToNot(HaveOccurred()) Expect(countingManager.getQueries).To(Equal(1)) - Expect(countingManager.listQueries).To(Equal(1)) + Expect(countingManager.listQueries).To(Equal(2)) By("updating Dataplane in store and waiting until cache invalidation") dp := core_mesh.NewDataplaneResource() @@ -126,7 +131,7 @@ var _ = Describe("ClusterLoadAssignment Cache", func() { cla, err = claCache.GetCLA(context.Background(), "mesh-0", "", envoy_common.NewCluster(envoy_common.WithService("backend")), envoy_common.APIV3) Expect(err).ToNot(HaveOccurred()) Expect(countingManager.getQueries).To(Equal(2)) - Expect(countingManager.listQueries).To(Equal(2)) + Expect(countingManager.listQueries).To(Equal(4)) expected, err = ioutil.ReadFile(filepath.Join("testdata", "cla.get.1.json")) Expect(err).ToNot(HaveOccurred()) From 9ffe50361dc9c8aa989cb5103c22036601d68a8a Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Sat, 12 Jun 2021 20:45:48 +0700 Subject: [PATCH 10/19] chore(kuma-cp) revert proxyId, fix e2e Signed-off-by: Ilya Lobkov --- pkg/core/xds/metadata.go | 14 +++++ pkg/core/xds/types.go | 63 +++---------------- pkg/core/xds/types_test.go | 10 +-- pkg/xds/auth/callbacks.go | 7 +-- pkg/xds/bootstrap/generator.go | 5 +- pkg/xds/bootstrap/parameters.go | 1 + pkg/xds/bootstrap/template_v3.go | 3 + .../v3/http_access_log_configurer_test.go | 2 +- .../v3/network_access_log_configurer_test.go | 2 +- pkg/xds/generator/dns_generator_test.go | 2 +- .../generator/inbound_proxy_generator_test.go | 2 +- pkg/xds/generator/ingress_generator_test.go | 2 +- .../outbound_proxy_generator_test.go | 4 +- .../prometheus_endpoint_generator_test.go | 20 +++--- .../proxy_template_profile_source_test.go | 2 +- .../proxy_template_raw_source_test.go | 18 +++--- pkg/xds/generator/proxy_template_test.go | 4 +- pkg/xds/generator/tracing_generator_test.go | 4 +- .../transparent_proxy_generator_test.go | 8 +-- .../server/callbacks/dataplane_lifecycle.go | 8 +-- .../callbacks/dataplane_status_tracker.go | 4 +- pkg/xds/server/v3/reconcile_test.go | 2 +- pkg/xds/server/v3/snapshot_generator_test.go | 2 +- pkg/xds/sync/components.go | 1 + pkg/xds/sync/dataplane_proxy_builder.go | 2 +- pkg/xds/sync/dataplane_watchdog.go | 17 +++-- pkg/xds/sync/ingress_proxy_builder.go | 5 +- test/framework/universal_app.go | 2 +- 28 files changed, 92 insertions(+), 124 deletions(-) diff --git a/pkg/core/xds/metadata.go b/pkg/core/xds/metadata.go index 8068779e0495..b2e367099717 100644 --- a/pkg/core/xds/metadata.go +++ b/pkg/core/xds/metadata.go @@ -5,6 +5,8 @@ import ( _struct "github.com/golang/protobuf/ptypes/struct" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/model/rest" @@ -23,6 +25,7 @@ const ( fieldDataplaneDNSEmptyPort = "dataplane.dns.empty.port" fieldDataplaneDataplaneResource = "dataplane.resource" fieldDynamicMetadata = "dynamicMetadata" + fieldDataplaneProxyType = "dataplane.proxyType" ) // DataplaneMetadata represents environment-specific part of a dataplane configuration. @@ -48,6 +51,7 @@ type DataplaneMetadata struct { DNSPort uint32 EmptyDNSPort uint32 DynamicMetadata map[string]string + ProxyType mesh_proto.DpType } func (m *DataplaneMetadata) GetDataplaneTokenPath() string { @@ -78,6 +82,13 @@ func (m *DataplaneMetadata) GetZoneIngressResource() *core_mesh.ZoneIngressResou return m.ZoneIngressResource } +func (m *DataplaneMetadata) GetProxyType() mesh_proto.DpType { + if m == nil || m.ProxyType == "" { + return mesh_proto.RegularDpType + } + return m.ProxyType +} + func (m *DataplaneMetadata) GetAdminPort() uint32 { if m == nil { return 0 @@ -117,6 +128,9 @@ func DataplaneMetadataFromXdsMetadata(xdsMetadata *_struct.Struct) *DataplaneMet if field := xdsMetadata.Fields[fieldDataplaneToken]; field != nil { metadata.DataplaneToken = field.GetStringValue() } + if field := xdsMetadata.Fields[fieldDataplaneProxyType]; field != nil { + metadata.ProxyType = mesh_proto.DpType(field.GetStringValue()) + } metadata.AdminPort = uint32Metadata(xdsMetadata, fieldDataplaneAdminPort) metadata.DNSPort = uint32Metadata(xdsMetadata, fieldDataplaneDNSPort) metadata.EmptyDNSPort = uint32Metadata(xdsMetadata, fieldDataplaneDNSEmptyPort) diff --git a/pkg/core/xds/types.go b/pkg/core/xds/types.go index 1d183585a1fc..ebc411c09b19 100644 --- a/pkg/core/xds/types.go +++ b/pkg/core/xds/types.go @@ -20,16 +20,12 @@ import ( type StreamID = int64 type ProxyId struct { - mesh string - name string - proxyType mesh_proto.DpType + mesh string + name string } func (id *ProxyId) String() string { - if id.proxyType == "" { - return fmt.Sprintf("%s.%s", id.mesh, id.name) - } - return fmt.Sprintf("%s.%s:%s", id.mesh, id.name, id.proxyType) + return fmt.Sprintf("%s.%s", id.mesh, id.name) } func (id *ProxyId) ToResourceKey() core_model.ResourceKey { @@ -39,13 +35,6 @@ func (id *ProxyId) ToResourceKey() core_model.ResourceKey { } } -func (id *ProxyId) Type() mesh_proto.DpType { - if id.proxyType == "" { - return mesh_proto.RegularDpType - } - return id.proxyType -} - // ServiceName is a convenience type alias to clarify the meaning of string value. type ServiceName = string @@ -219,51 +208,20 @@ func (l EndpointList) Filter(selector mesh_proto.TagSelector) EndpointList { return endpoints } -func BuildProxyId(mesh, name string, proxyType mesh_proto.DpType) *ProxyId { +func BuildProxyId(mesh, name string) *ProxyId { return &ProxyId{ - name: name, - mesh: mesh, - proxyType: proxyType, + name: name, + mesh: mesh, } } func ParseProxyIdFromString(id string) (*ProxyId, error) { - if id == "" { - return nil, errors.Errorf("Envoy ID must not be nil") - } - parts := strings.SplitN(id, ":", 2) - if len(parts) == 1 { - proxyId, err := ParseProxyIdFromStringOldFormat(id) - if err != nil { - return nil, err - } - return proxyId, nil - } - proxyType := mesh_proto.DpType(parts[1]) - meshAndName := strings.SplitN(parts[0], ".", 2) - mesh, name := meshAndName[0], meshAndName[1] - if name == "" { - return nil, errors.New("name must not be empty") - } - if mesh == "" && proxyType != mesh_proto.IngressDpType { - return nil, errors.New("mesh must not be empty") - } - return &ProxyId{ - mesh: mesh, - name: name, - proxyType: proxyType, - }, nil -} - -func ParseProxyIdFromStringOldFormat(id string) (*ProxyId, error) { if id == "" { return nil, errors.Errorf("Envoy ID must not be nil") } parts := strings.SplitN(id, ".", 2) mesh := parts[0] - if mesh == "" { - return nil, errors.New("mesh must not be empty") - } + // when proxy is an ingress mesh is empty if len(parts) < 2 { return nil, errors.New("the name should be provided after the dot") } @@ -277,10 +235,9 @@ func ParseProxyIdFromStringOldFormat(id string) (*ProxyId, error) { }, nil } -func FromResourceKey(proxyType mesh_proto.DpType, key core_model.ResourceKey) ProxyId { +func FromResourceKey(key core_model.ResourceKey) ProxyId { return ProxyId{ - mesh: key.Mesh, - name: key.Name, - proxyType: proxyType, + mesh: key.Mesh, + name: key.Name, } } diff --git a/pkg/core/xds/types_test.go b/pkg/core/xds/types_test.go index 59f6d78918a8..a49397fa82bc 100644 --- a/pkg/core/xds/types_test.go +++ b/pkg/core/xds/types_test.go @@ -31,19 +31,19 @@ var _ = Describe("xDS", func() { }, Entry("mesh and name without namespace", testCase{ nodeID: "demo.example", - expected: *core_xds.BuildProxyId("demo", "example", ""), + expected: *core_xds.BuildProxyId("demo", "example"), }), Entry("name with namespace and mesh", testCase{ nodeID: "demo.example.sample", - expected: *core_xds.BuildProxyId("demo", "example.sample", ""), + expected: *core_xds.BuildProxyId("demo", "example.sample"), }), Entry("mesh and name without namespace and proxy type", testCase{ nodeID: "demo.example:ingress", - expected: *core_xds.BuildProxyId("demo", "example", mesh_proto.IngressDpType), + expected: *core_xds.BuildProxyId("demo", "example"), }), Entry("name with namespace and mesh and proxy type", testCase{ nodeID: "demo.example.sample:ingress", - expected: *core_xds.BuildProxyId("demo", "example.sample", mesh_proto.IngressDpType), + expected: *core_xds.BuildProxyId("demo", "example.sample"), }), ) }) @@ -83,7 +83,7 @@ var _ = Describe("xDS", func() { Describe("ProxyId(...).ToResourceKey()", func() { It("should convert proxy ID to resource key", func() { // given - id := *core_xds.BuildProxyId("default", "demo", "") + id := *core_xds.BuildProxyId("default", "demo") // when key := id.ToResourceKey() diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index 0927511ee7e3..310e500a8171 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -118,11 +118,8 @@ func (a *authCallbacks) credential(streamID core_xds.StreamID) (Credential, erro } func (a *authCallbacks) authenticate(credential Credential, req util_xds.DiscoveryRequest) error { - proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()) - if err != nil { - return errors.Wrap(err, "request must have a valid Proxy Id") - } - switch proxyId.Type() { + md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) + switch md.GetProxyType() { case mesh_proto.IngressDpType: return a.authenticateZoneIngress(credential, req) default: diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 4f8cd93ac08e..ae201b4e5be3 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -85,7 +85,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr switch proxyType { case mesh_proto.IngressDpType: - proxyId := core_xds.BuildProxyId(request.Mesh, request.Name, mesh_proto.IngressDpType) + proxyId := core_xds.BuildProxyId(request.Mesh, request.Name) zoneIngress, err := b.zoneIngressFor(ctx, request, proxyId) if err != nil { return nil, "", err @@ -97,7 +97,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr b.hdsEnabled = false return b.generateFor(*proxyId, request, "ingress", adminPort) case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: - proxyId := core_xds.BuildProxyId(request.Mesh, request.Name, mesh_proto.RegularDpType) + proxyId := core_xds.BuildProxyId(request.Mesh, request.Name) dataplane, err := b.dataplaneFor(ctx, request, proxyId) if err != nil { return nil, "", err @@ -294,6 +294,7 @@ func (b *bootstrapGenerator) generateFor(proxyId core_xds.ProxyId, request types DynamicMetadata: request.DynamicMetadata, DNSPort: request.DNSPort, EmptyDNSPort: request.EmptyDNSPort, + ProxyType: request.ProxyType, } log.WithValues("params", params).Info("Generating bootstrap config") return b.configForParameters(params, request.BootstrapVersion) diff --git a/pkg/xds/bootstrap/parameters.go b/pkg/xds/bootstrap/parameters.go index 89adca4ed59c..1c99fefa1af0 100644 --- a/pkg/xds/bootstrap/parameters.go +++ b/pkg/xds/bootstrap/parameters.go @@ -28,4 +28,5 @@ type configParameters struct { DynamicMetadata map[string]string DNSPort uint32 EmptyDNSPort uint32 + ProxyType string } diff --git a/pkg/xds/bootstrap/template_v3.go b/pkg/xds/bootstrap/template_v3.go index c5d7728b5e68..02435da2cfe0 100644 --- a/pkg/xds/bootstrap/template_v3.go +++ b/pkg/xds/bootstrap/template_v3.go @@ -22,6 +22,9 @@ node: {{ end }} {{if .EmptyDNSPort }} dataplane.dns.empty.port: "{{ .EmptyDNSPort }}" +{{ end }} +{{if .ProxyType }} + dataplane.proxyType: "{{ .ProxyType }}" {{ end }} version: kumaDp: diff --git a/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go b/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go index dde7cbab85b4..f456545a1435 100644 --- a/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go +++ b/pkg/xds/envoy/listeners/v3/http_access_log_configurer_test.go @@ -34,7 +34,7 @@ var _ = Describe("HttpAccessLogConfigurer", func() { sourceService := "web" destinationService := "backend" proxy := &core_xds.Proxy{ - Id: *core_xds.BuildProxyId("web", "example", ""), + Id: *core_xds.BuildProxyId("web", "example"), Dataplane: &mesh_core.DataplaneResource{ Spec: &mesh_proto.Dataplane{ Networking: &mesh_proto.Dataplane_Networking{ diff --git a/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go b/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go index e00a8ff10c16..736088241c01 100644 --- a/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go +++ b/pkg/xds/envoy/listeners/v3/network_access_log_configurer_test.go @@ -34,7 +34,7 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { sourceService := "backend" destinationService := "db" proxy := &core_xds.Proxy{ - Id: *core_xds.BuildProxyId("example", "backend", ""), + Id: *core_xds.BuildProxyId("example", "backend"), Dataplane: &mesh_core.DataplaneResource{ Spec: &mesh_proto.Dataplane{ Networking: &mesh_proto.Dataplane_Networking{ diff --git a/pkg/xds/generator/dns_generator_test.go b/pkg/xds/generator/dns_generator_test.go index 598ee9f83a28..3092038df037 100644 --- a/pkg/xds/generator/dns_generator_test.go +++ b/pkg/xds/generator/dns_generator_test.go @@ -70,7 +70,7 @@ var _ = Describe("DNSGenerator", func() { Expect(err).ToNot(HaveOccurred()) Expect(util_proto.FromYAML(dpBytes, &dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "1", diff --git a/pkg/xds/generator/inbound_proxy_generator_test.go b/pkg/xds/generator/inbound_proxy_generator_test.go index a89475ec41ed..42e523e119da 100644 --- a/pkg/xds/generator/inbound_proxy_generator_test.go +++ b/pkg/xds/generator/inbound_proxy_generator_test.go @@ -64,7 +64,7 @@ var _ = Describe("InboundProxyGenerator", func() { Expect(err).ToNot(HaveOccurred()) Expect(util_proto.FromYAML(dpBytes, &dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "1", diff --git a/pkg/xds/generator/ingress_generator_test.go b/pkg/xds/generator/ingress_generator_test.go index 791dd8f5d077..88a354b15037 100644 --- a/pkg/xds/generator/ingress_generator_test.go +++ b/pkg/xds/generator/ingress_generator_test.go @@ -43,7 +43,7 @@ var _ = Describe("IngressGenerator", func() { }) Expect(err).ToNot(HaveOccurred()) proxy := &core_xds.Proxy{ - Id: *core_xds.BuildProxyId("default", "ingress", ""), + Id: *core_xds.BuildProxyId("default", "ingress"), ZoneIngress: zoneIngress, APIVersion: envoy_common.APIV3, Routing: core_xds.Routing{ diff --git a/pkg/xds/generator/outbound_proxy_generator_test.go b/pkg/xds/generator/outbound_proxy_generator_test.go index 638a68e08b39..4310ac89e3f4 100644 --- a/pkg/xds/generator/outbound_proxy_generator_test.go +++ b/pkg/xds/generator/outbound_proxy_generator_test.go @@ -159,7 +159,7 @@ var _ = Describe("OutboundProxyGenerator", func() { }, } proxy := &model.Proxy{ - Id: *model.BuildProxyId("default", "side-car", ""), + Id: *model.BuildProxyId("default", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "dp-1", @@ -500,7 +500,7 @@ var _ = Describe("OutboundProxyGenerator", func() { }, } proxy := &model.Proxy{ - Id: *model.BuildProxyId("default", "side-car", ""), + Id: *model.BuildProxyId("default", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "1", diff --git a/pkg/xds/generator/prometheus_endpoint_generator_test.go b/pkg/xds/generator/prometheus_endpoint_generator_test.go index e1d22b7eab77..cabd5663114b 100644 --- a/pkg/xds/generator/prometheus_endpoint_generator_test.go +++ b/pkg/xds/generator/prometheus_endpoint_generator_test.go @@ -52,7 +52,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", @@ -75,7 +75,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -121,7 +121,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -168,7 +168,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -243,7 +243,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -288,7 +288,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -360,7 +360,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -423,7 +423,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -487,7 +487,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, }, proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -542,7 +542,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { }, } proxy := &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ diff --git a/pkg/xds/generator/proxy_template_profile_source_test.go b/pkg/xds/generator/proxy_template_profile_source_test.go index 467c71f50b03..8d642d3d7736 100644 --- a/pkg/xds/generator/proxy_template_profile_source_test.go +++ b/pkg/xds/generator/proxy_template_profile_source_test.go @@ -99,7 +99,7 @@ var _ = Describe("ProxyTemplateProfileSource", func() { Expect(util_proto.FromYAML([]byte(given.dataplane), dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", diff --git a/pkg/xds/generator/proxy_template_raw_source_test.go b/pkg/xds/generator/proxy_template_raw_source_test.go index b862e9fc2f0f..cfb7965a00b2 100644 --- a/pkg/xds/generator/proxy_template_raw_source_test.go +++ b/pkg/xds/generator/proxy_template_raw_source_test.go @@ -43,7 +43,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }, Entry("should fail when `resource` field is empty", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -72,7 +72,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field is neither a YAML nor a JSON", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -100,7 +100,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field has unknown @type", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -130,7 +130,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field is a YAML without '@type' field", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -171,7 +171,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should fail when `resource` field is an invalid xDS resource", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -247,7 +247,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }, Entry("should support empty resource list", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -271,7 +271,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should support Listener resource as YAML", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -332,7 +332,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should support Cluster resource as YAML", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -391,7 +391,7 @@ var _ = Describe("ProxyTemplateRawSource", func() { }), Entry("should support Cluster resource as JSON", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", diff --git a/pkg/xds/generator/proxy_template_test.go b/pkg/xds/generator/proxy_template_test.go index 0402e29d58cf..4552e6d978f8 100644 --- a/pkg/xds/generator/proxy_template_test.go +++ b/pkg/xds/generator/proxy_template_test.go @@ -60,7 +60,7 @@ var _ = Describe("ProxyTemplateGenerator", func() { }, Entry("should fail when raw xDS resource is not valid", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", @@ -152,7 +152,7 @@ var _ = Describe("ProxyTemplateGenerator", func() { dataplane := &mesh_proto.Dataplane{} Expect(util_proto.FromYAML([]byte(given.dataplane), dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", diff --git a/pkg/xds/generator/tracing_generator_test.go b/pkg/xds/generator/tracing_generator_test.go index 9e158e831485..1f88b975007e 100644 --- a/pkg/xds/generator/tracing_generator_test.go +++ b/pkg/xds/generator/tracing_generator_test.go @@ -41,7 +41,7 @@ var _ = Describe("TracingProxyGenerator", func() { }, Entry("Mesh has no Tracing configuration", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", @@ -74,7 +74,7 @@ var _ = Describe("TracingProxyGenerator", func() { }, Entry("should create cluster for Zipkin", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "demo.backend-01", ""), + Id: *model.BuildProxyId("", "demo.backend-01"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Name: "backend-01", diff --git a/pkg/xds/generator/transparent_proxy_generator_test.go b/pkg/xds/generator/transparent_proxy_generator_test.go index 613b0d3aa22c..8f0627473dd0 100644 --- a/pkg/xds/generator/transparent_proxy_generator_test.go +++ b/pkg/xds/generator/transparent_proxy_generator_test.go @@ -57,7 +57,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }, Entry("transparent_proxying=false", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -69,7 +69,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }), Entry("transparent_proxying=true", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -100,7 +100,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }), Entry("transparent_proxying=true with logs", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", @@ -131,7 +131,7 @@ var _ = Describe("TransparentProxyGenerator", func() { }), Entry("transparent_proxying=true ipv6", testCase{ proxy: &model.Proxy{ - Id: *model.BuildProxyId("", "side-car", ""), + Id: *model.BuildProxyId("", "side-car"), Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ Version: "v1", diff --git a/pkg/xds/server/callbacks/dataplane_lifecycle.go b/pkg/xds/server/callbacks/dataplane_lifecycle.go index 5ce14e483d17..494810586d57 100644 --- a/pkg/xds/server/callbacks/dataplane_lifecycle.go +++ b/pkg/xds/server/callbacks/dataplane_lifecycle.go @@ -103,12 +103,8 @@ func (d *DataplaneLifecycle) OnStreamRequest(streamID int64, request util_xds.Di defer d.Unlock() md := xds.DataplaneMetadataFromXdsMetadata(request.Metadata()) - proxyId, err := xds.ParseProxyIdFromString(request.NodeId()) - if err != nil { - return nil - } - if proxyId.Type() == mesh_proto.RegularDpType && md.GetDataplaneResource() != nil { + if md.GetProxyType() == mesh_proto.RegularDpType && md.GetDataplaneResource() != nil { lifecycleLog.Info("registering dataplane", "dataplane", md.GetDataplaneResource(), "streamID", streamID, "nodeID", request.NodeId()) if err := d.registerDataplane(md.GetDataplaneResource()); err != nil { return errors.Wrap(err, "could not register dataplane passed in kuma-dp run") @@ -119,7 +115,7 @@ func (d *DataplaneLifecycle) OnStreamRequest(streamID int64, request util_xds.Di return nil } - if proxyId.Type() == mesh_proto.IngressDpType && md.ZoneIngressResource != nil { + if md.GetProxyType() == mesh_proto.IngressDpType && md.ZoneIngressResource != nil { lifecycleLog.Info("registering zone ingress", "zoneIngress", md.ZoneIngressResource, "streamID", streamID, "nodeID", request.NodeId()) if err := d.registerZoneIngress(md.ZoneIngressResource); err != nil { return errors.Wrap(err, "could not register zone ingress passed in kuma-dp run") diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index 3a7009521e69..5a7a3380f799 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -114,6 +114,8 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di state.mu.Lock() // write access to the per Dataplane info defer state.mu.Unlock() + md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) + // infer Dataplane id if state.dataplaneId == (core_model.ResourceKey{}) { if proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { @@ -122,7 +124,7 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di statusTrackerLog.Error(err, "failed to extract version out of the Envoy metadata", "streamid", streamID, "metadata", req.Metadata()) } // kick off async Dataplane status flusher - if proxyId.Type() != mesh_proto.IngressDpType { + if md.GetProxyType() != mesh_proto.IngressDpType { go c.createStatusSink(state).Start(state.stop) } } else { diff --git a/pkg/xds/server/v3/reconcile_test.go b/pkg/xds/server/v3/reconcile_test.go index 3f07e51d963d..abeb0a149f33 100644 --- a/pkg/xds/server/v3/reconcile_test.go +++ b/pkg/xds/server/v3/reconcile_test.go @@ -115,7 +115,7 @@ var _ = Describe("Reconcile", func() { By("simulating discovery event") // when proxy := &xds_model.Proxy{ - Id: *xds_model.BuildProxyId("demo", "example", ""), + Id: *xds_model.BuildProxyId("demo", "example"), Dataplane: dataplane, } err := r.Reconcile(xds_context.Context{}, proxy) diff --git a/pkg/xds/server/v3/snapshot_generator_test.go b/pkg/xds/server/v3/snapshot_generator_test.go index f116f0b6fc66..3a0ee1fb501f 100644 --- a/pkg/xds/server/v3/snapshot_generator_test.go +++ b/pkg/xds/server/v3/snapshot_generator_test.go @@ -67,7 +67,7 @@ var _ = Describe("Reconcile", func() { Expect(util_proto.FromYAML(dpBytes, &dataplane)).To(Succeed()) proxy := &model.Proxy{ - Id: *model.BuildProxyId("", "demo.web1", ""), + Id: *model.BuildProxyId("", "demo.web1"), APIVersion: envoy_common.APIV3, Dataplane: &mesh_core.DataplaneResource{ Meta: &test_model.ResourceMeta{ diff --git a/pkg/xds/sync/components.go b/pkg/xds/sync/components.go index dc5da13db637..adf22a32b09d 100644 --- a/pkg/xds/sync/components.go +++ b/pkg/xds/sync/components.go @@ -64,6 +64,7 @@ func DefaultDataplaneWatchdogFactory( ingressReconciler: ingressReconciler, xdsContextBuilder: xdsContextBuilder, meshCache: meshSnapshotCache, + metadataTracker: metadataTracker, } return NewDataplaneWatchdogFactory( xdsMetrics, diff --git a/pkg/xds/sync/dataplane_proxy_builder.go b/pkg/xds/sync/dataplane_proxy_builder.go index c73d2e591a9a..f25e00b0a69f 100644 --- a/pkg/xds/sync/dataplane_proxy_builder.go +++ b/pkg/xds/sync/dataplane_proxy_builder.go @@ -55,7 +55,7 @@ func (p *DataplaneProxyBuilder) build(key core_model.ResourceKey, streamId int64 } proxy := &xds.Proxy{ - Id: xds.FromResourceKey(mesh_proto.RegularDpType, key), + Id: xds.FromResourceKey(key), APIVersion: p.apiVersion, Dataplane: dp, Metadata: p.MetadataTracker.Metadata(streamId), diff --git a/pkg/xds/sync/dataplane_watchdog.go b/pkg/xds/sync/dataplane_watchdog.go index 92f5eeadffb8..432a0f1e5ef9 100644 --- a/pkg/xds/sync/dataplane_watchdog.go +++ b/pkg/xds/sync/dataplane_watchdog.go @@ -24,6 +24,7 @@ type DataplaneWatchdogDependencies struct { ingressReconciler SnapshotReconciler xdsContextBuilder *xdsContextBuilder meshCache *mesh.Cache + metadataTracker DataplaneMetadataTracker } type DataplaneWatchdog struct { @@ -43,8 +44,7 @@ func NewDataplaneWatchdog(deps DataplaneWatchdogDependencies, proxyId *core_xds. DataplaneWatchdogDependencies: deps, key: proxyId.ToResourceKey(), streamId: streamId, - log: core.Log.WithValues("key", "key", "streamID", streamId), - dpType: proxyId.Type(), + log: core.Log.WithValues("key", proxyId.ToResourceKey(), "streamID", streamId), proxyTypeSettled: false, } } @@ -52,6 +52,9 @@ func NewDataplaneWatchdog(deps DataplaneWatchdogDependencies, proxyId *core_xds. func (d *DataplaneWatchdog) Sync() error { ctx := context.Background() + if d.dpType == "" { + d.dpType = d.metadataTracker.Metadata(d.streamId).GetProxyType() + } // backwards compatibility if d.dpType == mesh_proto.RegularDpType && !d.proxyTypeSettled { dataplane := mesh_core.NewDataplaneResource() @@ -75,7 +78,7 @@ func (d *DataplaneWatchdog) Sync() error { } func (d *DataplaneWatchdog) Cleanup() error { - proxyID := core_xds.FromResourceKey(d.dpType, d.key) + proxyID := core_xds.FromResourceKey(d.key) switch d.dpType { case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: return d.dataplaneReconciler.Clear(&proxyID) @@ -117,13 +120,7 @@ func (d *DataplaneWatchdog) syncDataplane() error { // syncIngress synces state of Ingress Dataplane. Notice that it does not use Mesh Hash yet because Ingress supports many Meshes. func (d *DataplaneWatchdog) syncIngress() error { envoyCtx := d.xdsContextBuilder.buildContext(d.streamId) - var proxyType mesh_proto.DpType - if d.proxyTypeSettled { - proxyType = mesh_proto.RegularDpType - } else { - proxyType = d.dpType - } - proxy, err := d.ingressProxyBuilder.build(proxyType, d.key, d.streamId) + proxy, err := d.ingressProxyBuilder.build(d.key, d.streamId) if err != nil { return err } diff --git a/pkg/xds/sync/ingress_proxy_builder.go b/pkg/xds/sync/ingress_proxy_builder.go index d897af051de3..54cf7edc6e7c 100644 --- a/pkg/xds/sync/ingress_proxy_builder.go +++ b/pkg/xds/sync/ingress_proxy_builder.go @@ -3,7 +3,6 @@ package sync import ( "context" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core/dns/lookup" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" @@ -24,7 +23,7 @@ type IngressProxyBuilder struct { apiVersion envoy.APIVersion } -func (p *IngressProxyBuilder) build(proxyType mesh_proto.DpType, key core_model.ResourceKey, streamId int64) (*xds.Proxy, error) { +func (p *IngressProxyBuilder) build(key core_model.ResourceKey, streamId int64) (*xds.Proxy, error) { ctx := context.Background() zoneIngress, err := p.getZoneIngress(key) @@ -48,7 +47,7 @@ func (p *IngressProxyBuilder) build(proxyType mesh_proto.DpType, key core_model. } proxy := &xds.Proxy{ - Id: xds.FromResourceKey(proxyType, key), + Id: xds.FromResourceKey(key), APIVersion: p.apiVersion, ZoneIngress: zoneIngress, Metadata: p.MetadataTracker.Metadata(streamId), diff --git a/test/framework/universal_app.go b/test/framework/universal_app.go index c3f89a4d03c7..a0de374a3e25 100644 --- a/test/framework/universal_app.go +++ b/test/framework/universal_app.go @@ -45,7 +45,7 @@ networking: ` ZoneIngress = ` type: ZoneIngress -name: new-ingress +name: ingress address: {{ address }} advertisedAddress: %s advertisedPort: %d From cc87cfefe305a4016037428d7cba6eeb13d04c34 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Sat, 12 Jun 2021 22:35:09 +0700 Subject: [PATCH 11/19] chore(kuma-cp) fix tests Signed-off-by: Ilya Lobkov --- pkg/core/xds/types_test.go | 4 ++-- pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml | 2 +- .../bootstrap/testdata/bootstrap.overridden.golden.yaml | 2 +- .../bootstrap/testdata/bootstrap.universal.golden.yaml | 2 +- .../generator.custom-config-minimal-request.golden.yaml | 2 +- .../testdata/generator.custom-config.golden.yaml | 2 +- .../generator.default-config-minimal-request.golden.yaml | 2 +- .../testdata/generator.default-config.golden.yaml | 2 +- .../generator.default-config.kubernetes.golden.yaml | 2 +- .../generator.default-config.kubernetes.ipv6.golden.yaml | 2 +- .../e2e/trafficroute/universal_multizone/traffic_route.go | 8 ++++++-- 11 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/core/xds/types_test.go b/pkg/core/xds/types_test.go index a49397fa82bc..18ccd36983a8 100644 --- a/pkg/core/xds/types_test.go +++ b/pkg/core/xds/types_test.go @@ -38,11 +38,11 @@ var _ = Describe("xDS", func() { expected: *core_xds.BuildProxyId("demo", "example.sample"), }), Entry("mesh and name without namespace and proxy type", testCase{ - nodeID: "demo.example:ingress", + nodeID: "demo.example", expected: *core_xds.BuildProxyId("demo", "example"), }), Entry("name with namespace and mesh and proxy type", testCase{ - nodeID: "demo.example.sample:ingress", + nodeID: "demo.example.sample", expected: *core_xds.BuildProxyId("demo", "example.sample"), }), ) diff --git a/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml b/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml index 289290d2a5ad..4c8916154c6b 100644 --- a/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml +++ b/pkg/xds/bootstrap/testdata/bootstrap.k8s.golden.yaml @@ -53,7 +53,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: default.dp-1.default:regular + id: default.dp-1.default metadata: dataplaneTokenPath: /tmp/token version: diff --git a/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml b/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml index 6282427d4a5d..2bdce08e32e5 100644 --- a/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml +++ b/pkg/xds/bootstrap/testdata/bootstrap.overridden.golden.yaml @@ -59,7 +59,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: default.dp-1.default:regular + id: default.dp-1.default metadata: dataplane.admin.port: "1234" dataplaneTokenPath: /tmp/token diff --git a/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml b/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml index 54fb0bb9455b..14b3eaddc870 100644 --- a/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml +++ b/pkg/xds/bootstrap/testdata/bootstrap.universal.golden.yaml @@ -53,7 +53,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: default.dp-1:regular + id: default.dp-1 metadata: dataplaneTokenPath: /tmp/token version: diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml index 5d436934e9e6..ceee99e20b10 100644 --- a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request.golden.yaml @@ -33,7 +33,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace:regular + id: mesh.name.namespace metadata: dataplane.admin.port: "9902" version: diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml index 597245208273..6fd8d99b4578 100644 --- a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml @@ -59,7 +59,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace:regular + id: mesh.name.namespace metadata: dataplane.admin.port: "1234" dataplane.resource: ' { "type": "Dataplane", "mesh": "mesh", "name": "name.namespace", diff --git a/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml index 0585aeb85f9b..6324d5b8f4bc 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config-minimal-request.golden.yaml @@ -27,7 +27,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace:regular + id: mesh.name.namespace metadata: version: envoy: diff --git a/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml index d4734834500a..746680dc8fd6 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config.golden.yaml @@ -39,7 +39,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace:regular + id: mesh.name.namespace metadata: dataplane.admin.port: "1234" dataplane.dns.empty.port: "53002" diff --git a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml index 86961947b297..f3329800641c 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.golden.yaml @@ -39,7 +39,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace:regular + id: mesh.name.namespace metadata: dataplane.admin.port: "1234" dataplaneTokenPath: /tmp/token diff --git a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml index c157fc37ff5b..8c9403bf9964 100644 --- a/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.default-config.kubernetes.ipv6.golden.yaml @@ -39,7 +39,7 @@ layeredRuntime: re2.max_program_size.warn_level: 1000 node: cluster: backend - id: mesh.name.namespace:regular + id: mesh.name.namespace metadata: dataplane.admin.port: "1234" dataplaneTokenPath: /tmp/token diff --git a/test/e2e/trafficroute/universal_multizone/traffic_route.go b/test/e2e/trafficroute/universal_multizone/traffic_route.go index 6a177c8463a8..2d1878d54494 100644 --- a/test/e2e/trafficroute/universal_multizone/traffic_route.go +++ b/test/e2e/trafficroute/universal_multizone/traffic_route.go @@ -57,7 +57,7 @@ routing: Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken(defaultMesh, "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateZoneIngressToken("new-ingress") + ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") Expect(err).ToNot(HaveOccurred()) // Cluster 1 @@ -144,7 +144,7 @@ routing: Expect(global.DismissCluster()).To(Succeed()) }) - It("should access all instances of the service", func() { + FIt("should access all instances of the service", func() { const trafficRoute = ` type: TrafficRoute name: three-way-route @@ -174,6 +174,10 @@ conf: ` Expect(YamlUniversal(trafficRoute)(global)).To(Succeed()) + //for { + // time.Sleep(1 * time.Hour) + //} + Eventually(func() (map[string]int, error) { return CollectResponsesByInstance(zone1, "demo-client", "test-server.mesh") }, "30s", "500ms").Should( From 64ef861e4d8f850a34a824e5a74cfe874eeb08aa Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Sat, 12 Jun 2021 22:38:40 +0700 Subject: [PATCH 12/19] chore(kuma-cp) make check Signed-off-by: Ilya Lobkov --- test/e2e/trafficroute/universal_multizone/traffic_route.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/e2e/trafficroute/universal_multizone/traffic_route.go b/test/e2e/trafficroute/universal_multizone/traffic_route.go index 2d1878d54494..f7192224d989 100644 --- a/test/e2e/trafficroute/universal_multizone/traffic_route.go +++ b/test/e2e/trafficroute/universal_multizone/traffic_route.go @@ -174,10 +174,6 @@ conf: ` Expect(YamlUniversal(trafficRoute)(global)).To(Succeed()) - //for { - // time.Sleep(1 * time.Hour) - //} - Eventually(func() (map[string]int, error) { return CollectResponsesByInstance(zone1, "demo-client", "test-server.mesh") }, "30s", "500ms").Should( From 2f0817aeb2a0d183133b28422a598ade5eeb6d6d Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Sat, 12 Jun 2021 23:53:15 +0700 Subject: [PATCH 13/19] chore(kuma-cp) make check Signed-off-by: Ilya Lobkov --- test/e2e/trafficroute/universal_multizone/traffic_route.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/trafficroute/universal_multizone/traffic_route.go b/test/e2e/trafficroute/universal_multizone/traffic_route.go index f7192224d989..36026cea3059 100644 --- a/test/e2e/trafficroute/universal_multizone/traffic_route.go +++ b/test/e2e/trafficroute/universal_multizone/traffic_route.go @@ -144,7 +144,7 @@ routing: Expect(global.DismissCluster()).To(Succeed()) }) - FIt("should access all instances of the service", func() { + It("should access all instances of the service", func() { const trafficRoute = ` type: TrafficRoute name: three-way-route From c0478e7f25531ab128ec080ea3cb340ab98e6338 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Mon, 14 Jun 2021 00:57:19 +0700 Subject: [PATCH 14/19] chore(kuma-cp) review Signed-off-by: Ilya Lobkov --- api/mesh/v1alpha1/dataplane_helpers.go | 28 ++- api/mesh/v1alpha1/zone_ingress.pb.go | 217 ++++++++++++------ api/mesh/v1alpha1/zone_ingress.proto | 35 +-- app/kuma-dp/cmd/run.go | 6 +- .../pkg/dataplane/envoy/remote_bootstrap.go | 8 +- .../dataplane/envoy/remote_bootstrap_test.go | 6 +- .../cmd/generate/generate_dataplane_token.go | 12 +- ...all-control-plane.with-ingress.golden.yaml | 22 +- app/kumactl/pkg/tokens/client_test.go | 2 +- deployments/charts/kuma/README.md | 1 - .../kuma/templates/ingress-deployment.yaml | 5 +- deployments/charts/kuma/values.yaml | 2 - docs/cmd/kumactl/HELP.md | 2 +- pkg/config/app/kuma-dp/config.go | 12 +- .../apis/mesh/dataplane_overview_helpers.go | 2 +- .../apis/mesh/zone_ingress_helpers.go | 12 +- .../apis/mesh/zone_ingress_validator.go | 18 +- .../apis/mesh/zone_ingress_validator_test.go | 39 ++-- pkg/core/xds/metadata.go | 8 +- pkg/dns/outbound_test.go | 83 +++++++ .../k8s/controllers/ingress_converter.go | 8 +- pkg/test/kds/samples/resources.go | 10 +- pkg/tokens/builtin/issuer/issuer.go | 18 +- .../server/types/zoneingress_token_request.go | 2 +- pkg/tokens/builtin/server/webservice.go | 11 +- pkg/tokens/builtin/server/webservice_test.go | 2 +- pkg/tokens/builtin/zoneingress/issuer.go | 8 +- pkg/xds/auth/callbacks.go | 6 +- pkg/xds/auth/callbacks_test.go | 67 +++++- pkg/xds/auth/components/components.go | 2 +- pkg/xds/auth/universal/auth_test.go | 8 +- pkg/xds/auth/universal/authenticator.go | 22 +- pkg/xds/bootstrap/generator.go | 8 +- pkg/xds/cache/mesh/snapshot.go | 4 +- pkg/xds/generator/admin_proxy_generator.go | 2 +- pkg/xds/generator/ingress_generator.go | 4 +- .../server/callbacks/dataplane_lifecycle.go | 16 +- .../callbacks/dataplane_status_tracker.go | 2 +- pkg/xds/sync/dataplane_watchdog.go | 14 +- pkg/xds/topology/dataplanes.go | 10 +- pkg/xds/topology/outbound.go | 6 +- test/e2e/deploy/kuma_deploy_universal.go | 10 +- .../healthcheck/hybrid/healthcheck_hybrid.go | 7 +- test/e2e/hybrid/kuma_hybrid.go | 10 +- test/e2e/hybrid/kuma_hybrid_kube_global.go | 6 +- .../hybrid/traffic_permission_hybrid.go | 6 +- .../universal_multizone/traffic_route.go | 10 +- test/framework/interface.go | 2 +- test/framework/k8s_controlplane.go | 4 +- test/framework/universal_app.go | 11 +- test/framework/universal_controlplane.go | 4 +- 51 files changed, 530 insertions(+), 290 deletions(-) diff --git a/api/mesh/v1alpha1/dataplane_helpers.go b/api/mesh/v1alpha1/dataplane_helpers.go index ca5fbedd44c2..c15b9e06f6f7 100644 --- a/api/mesh/v1alpha1/dataplane_helpers.go +++ b/api/mesh/v1alpha1/dataplane_helpers.go @@ -30,24 +30,34 @@ const ( // External service tag ExternalServiceTag = "kuma.io/external-service-name" - RegularDpType DpType = "regular" - IngressDpType DpType = "ingress" - GatewayDpType DpType = "gateway" - // Used for Service-less dataplanes TCPPortReserved = 49151 // IANA Reserved ) -type DpType string +type ProxyType string + +const ( + DataplaneProxyType ProxyType = "dataplane" + IngressProxyType ProxyType = "ingress" + GatewayProxyType ProxyType = "gateway" +) + +func (t ProxyType) IsValid() error { + switch t { + case DataplaneProxyType, IngressProxyType, GatewayProxyType: + return nil + } + return errors.New("Invalid proxy type") +} -func (d *Dataplane) DpType() DpType { +func (d *Dataplane) ProxyType() ProxyType { if d.IsIngress() { - return IngressDpType + return IngressProxyType } if d.GetNetworking().GetGateway() != nil { - return GatewayDpType + return GatewayProxyType } - return RegularDpType + return DataplaneProxyType } type InboundInterface struct { diff --git a/api/mesh/v1alpha1/zone_ingress.pb.go b/api/mesh/v1alpha1/zone_ingress.pb.go index 10c3808845f2..7e8e0e72dc29 100644 --- a/api/mesh/v1alpha1/zone_ingress.pb.go +++ b/api/mesh/v1alpha1/zone_ingress.pb.go @@ -36,22 +36,15 @@ type ZoneIngress struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Address on which inbound listener will be exposed - Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` - // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible - // to other Kuma clusters. - AdvertisedAddress string `protobuf:"bytes,2,opt,name=advertisedAddress,proto3" json:"advertisedAddress,omitempty"` - // Port of the inbound interface that will forward requests to the service. - Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` - // AdvertisedPort defines port on which ZoneIngress is accessible to other - // Kuma clusters. - AdvertisedPort uint32 `protobuf:"varint,4,opt,name=advertisedPort,proto3" json:"advertisedPort,omitempty"` // Zone field contains Zone name where ingress is serving, field will be // automatically set by Global Kuma CP - Zone string `protobuf:"bytes,6,opt,name=zone,proto3" json:"zone,omitempty"` + Zone string `protobuf:"bytes,1,opt,name=zone,proto3" json:"zone,omitempty"` + // Networking defines the address and port of the Ingress to listen on. + // Additionally publicly advertised address and port could be specified. + Networking *ZoneIngress_Networking `protobuf:"bytes,2,opt,name=networking,proto3" json:"networking,omitempty"` // AvailableService contains tags that represent unique subset of // endpoints - AvailableServices []*ZoneIngress_AvailableService `protobuf:"bytes,5,rep,name=availableServices,proto3" json:"availableServices,omitempty"` + AvailableServices []*ZoneIngress_AvailableService `protobuf:"bytes,3,rep,name=availableServices,proto3" json:"availableServices,omitempty"` } func (x *ZoneIngress) Reset() { @@ -86,46 +79,102 @@ func (*ZoneIngress) Descriptor() ([]byte, []int) { return file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP(), []int{0} } -func (x *ZoneIngress) GetAddress() string { +func (x *ZoneIngress) GetZone() string { if x != nil { - return x.Address + return x.Zone } return "" } -func (x *ZoneIngress) GetAdvertisedAddress() string { +func (x *ZoneIngress) GetNetworking() *ZoneIngress_Networking { if x != nil { - return x.AdvertisedAddress + return x.Networking } - return "" + return nil } -func (x *ZoneIngress) GetPort() uint32 { +func (x *ZoneIngress) GetAvailableServices() []*ZoneIngress_AvailableService { if x != nil { - return x.Port + return x.AvailableServices } - return 0 + return nil +} + +type ZoneIngress_Networking struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Address on which inbound listener will be exposed + Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` + // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible + // to other Kuma clusters. + AdvertisedAddress string `protobuf:"bytes,2,opt,name=advertisedAddress,proto3" json:"advertisedAddress,omitempty"` + // Port of the inbound interface that will forward requests to the service. + Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + // AdvertisedPort defines port on which ZoneIngress is accessible to other + // Kuma clusters. + AdvertisedPort uint32 `protobuf:"varint,4,opt,name=advertisedPort,proto3" json:"advertisedPort,omitempty"` +} + +func (x *ZoneIngress_Networking) Reset() { + *x = ZoneIngress_Networking{} + if protoimpl.UnsafeEnabled { + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ZoneIngress_Networking) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *ZoneIngress) GetAdvertisedPort() uint32 { +func (*ZoneIngress_Networking) ProtoMessage() {} + +func (x *ZoneIngress_Networking) ProtoReflect() protoreflect.Message { + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ZoneIngress_Networking.ProtoReflect.Descriptor instead. +func (*ZoneIngress_Networking) Descriptor() ([]byte, []int) { + return file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *ZoneIngress_Networking) GetAddress() string { if x != nil { - return x.AdvertisedPort + return x.Address } - return 0 + return "" } -func (x *ZoneIngress) GetZone() string { +func (x *ZoneIngress_Networking) GetAdvertisedAddress() string { if x != nil { - return x.Zone + return x.AdvertisedAddress } return "" } -func (x *ZoneIngress) GetAvailableServices() []*ZoneIngress_AvailableService { +func (x *ZoneIngress_Networking) GetPort() uint32 { if x != nil { - return x.AvailableServices + return x.Port } - return nil + return 0 +} + +func (x *ZoneIngress_Networking) GetAdvertisedPort() uint32 { + if x != nil { + return x.AdvertisedPort + } + return 0 } type ZoneIngress_AvailableService struct { @@ -144,7 +193,7 @@ type ZoneIngress_AvailableService struct { func (x *ZoneIngress_AvailableService) Reset() { *x = ZoneIngress_AvailableService{} if protoimpl.UnsafeEnabled { - mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1] + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -157,7 +206,7 @@ func (x *ZoneIngress_AvailableService) String() string { func (*ZoneIngress_AvailableService) ProtoMessage() {} func (x *ZoneIngress_AvailableService) ProtoReflect() protoreflect.Message { - mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1] + mi := &file_mesh_v1alpha1_zone_ingress_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -170,7 +219,7 @@ func (x *ZoneIngress_AvailableService) ProtoReflect() protoreflect.Message { // Deprecated: Use ZoneIngress_AvailableService.ProtoReflect.Descriptor instead. func (*ZoneIngress_AvailableService) Descriptor() ([]byte, []int) { - return file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP(), []int{0, 0} + return file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP(), []int{0, 1} } func (x *ZoneIngress_AvailableService) GetTags() map[string]string { @@ -207,40 +256,46 @@ var file_mesh_v1alpha1_zone_ingress_proto_rawDesc = []byte{ 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, - 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd5, 0x03, 0x0a, - 0x0b, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, - 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, - 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, - 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x64, 0x76, 0x65, - 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x5e, 0x0a, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, - 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x52, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x1a, 0xcd, 0x01, 0x0a, 0x10, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, - 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x61, 0x67, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb0, 0x04, 0x0a, + 0x0b, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, + 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, + 0x12, 0x4a, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, + 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, + 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x5e, 0x0a, 0x11, + 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, - 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x1a, 0x37, 0x0a, 0x09, 0x54, - 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, - 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, + 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x1a, 0x90, 0x01, 0x0a, + 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, + 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, + 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x1a, + 0xcd, 0x01, 0x0a, 0x10, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, + 0x74, 0x61, 0x67, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, + 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, + 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, + 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, + 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -255,20 +310,22 @@ func file_mesh_v1alpha1_zone_ingress_proto_rawDescGZIP() []byte { return file_mesh_v1alpha1_zone_ingress_proto_rawDescData } -var file_mesh_v1alpha1_zone_ingress_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_mesh_v1alpha1_zone_ingress_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_mesh_v1alpha1_zone_ingress_proto_goTypes = []interface{}{ (*ZoneIngress)(nil), // 0: kuma.mesh.v1alpha1.ZoneIngress - (*ZoneIngress_AvailableService)(nil), // 1: kuma.mesh.v1alpha1.ZoneIngress.AvailableService - nil, // 2: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry + (*ZoneIngress_Networking)(nil), // 1: kuma.mesh.v1alpha1.ZoneIngress.Networking + (*ZoneIngress_AvailableService)(nil), // 2: kuma.mesh.v1alpha1.ZoneIngress.AvailableService + nil, // 3: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry } var file_mesh_v1alpha1_zone_ingress_proto_depIdxs = []int32{ - 1, // 0: kuma.mesh.v1alpha1.ZoneIngress.availableServices:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService - 2, // 1: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.tags:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 1, // 0: kuma.mesh.v1alpha1.ZoneIngress.networking:type_name -> kuma.mesh.v1alpha1.ZoneIngress.Networking + 2, // 1: kuma.mesh.v1alpha1.ZoneIngress.availableServices:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService + 3, // 2: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.tags:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_mesh_v1alpha1_zone_ingress_proto_init() } @@ -291,6 +348,18 @@ func file_mesh_v1alpha1_zone_ingress_proto_init() { } } file_mesh_v1alpha1_zone_ingress_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ZoneIngress_Networking); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_mesh_v1alpha1_zone_ingress_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ZoneIngress_AvailableService); i { case 0: return &v.state @@ -309,7 +378,7 @@ func file_mesh_v1alpha1_zone_ingress_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_mesh_v1alpha1_zone_ingress_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/api/mesh/v1alpha1/zone_ingress.proto b/api/mesh/v1alpha1/zone_ingress.proto index b042ac59671f..d4f94a77e9f4 100644 --- a/api/mesh/v1alpha1/zone_ingress.proto +++ b/api/mesh/v1alpha1/zone_ingress.proto @@ -13,23 +13,30 @@ import "validate/validate.proto"; // mode, dataplane has only inbound interfaces. Every inbound interface matches // with services that reside in that cluster. message ZoneIngress { - // Address on which inbound listener will be exposed - string address = 1; - // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible - // to other Kuma clusters. - string advertisedAddress = 2; + // Zone field contains Zone name where ingress is serving, field will be + // automatically set by Global Kuma CP + string zone = 1; - // Port of the inbound interface that will forward requests to the service. - uint32 port = 3; + message Networking { + // Address on which inbound listener will be exposed + string address = 1; - // AdvertisedPort defines port on which ZoneIngress is accessible to other - // Kuma clusters. - uint32 advertisedPort = 4; + // AdvertisedAddress defines IP or DNS name on which ZoneIngress is + // accessible to other Kuma clusters. + string advertisedAddress = 2; - // Zone field contains Zone name where ingress is serving, field will be - // automatically set by Global Kuma CP - string zone = 6; + // Port of the inbound interface that will forward requests to the service. + uint32 port = 3; + + // AdvertisedPort defines port on which ZoneIngress is accessible to other + // Kuma clusters. + uint32 advertisedPort = 4; + } + + // Networking defines the address and port of the Ingress to listen on. + // Additionally publicly advertised address and port could be specified. + Networking networking = 2; message AvailableService { // tags of the service @@ -42,5 +49,5 @@ message ZoneIngress { // AvailableService contains tags that represent unique subset of // endpoints - repeated AvailableService availableServices = 5; + repeated AvailableService availableServices = 3; } diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index db85f3594ba9..53ac264487fa 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -8,6 +8,8 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + kumadp_config "github.com/kumahq/kuma/app/kuma-dp/pkg/config" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/dnsserver" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics" @@ -55,7 +57,7 @@ func newRunCmd(rootCtx *RootContext) *cobra.Command { } var err error - if cfg.Dataplane.IsIngress { + if mesh_proto.ProxyType(cfg.Dataplane.ProxyType) == mesh_proto.IngressProxyType { proxyResource, err = readZoneIngressResource(cmd, cfg) if err != nil { runLog.Error(err, "unable to read provided zone ingress") @@ -200,7 +202,7 @@ func newRunCmd(rootCtx *RootContext) *cobra.Command { cmd.PersistentFlags().StringVar(&cfg.Dataplane.Name, "name", cfg.Dataplane.Name, "Name of the Dataplane") cmd.PersistentFlags().Var(&cfg.Dataplane.AdminPort, "admin-port", `Port (or range of ports to choose from) for Envoy Admin API to listen on. Empty value indicates that Envoy Admin API should not be exposed over TCP. Format: "9901 | 9901-9999 | 9901- | -9901"`) cmd.PersistentFlags().StringVar(&cfg.Dataplane.Mesh, "mesh", cfg.Dataplane.Mesh, "Mesh that Dataplane belongs to") - cmd.PersistentFlags().BoolVar(&cfg.Dataplane.IsIngress, "ingress", false, "If true then kuma-dp will register itself as a ZoneIngress") + cmd.PersistentFlags().StringVar(&cfg.Dataplane.ProxyType, "proxy-type", "dataplane", `type of the Dataplane ("dataplane", "ingress")`) cmd.PersistentFlags().StringVar(&cfg.ControlPlane.URL, "cp-address", cfg.ControlPlane.URL, "URL of the Control Plane Dataplane Server. Example: https://localhost:5678") cmd.PersistentFlags().StringVar(&cfg.ControlPlane.CaCertFile, "ca-cert-file", cfg.ControlPlane.CaCertFile, "Path to CA cert by which connection to the Control Plane will be verified if HTTPS is used") cmd.PersistentFlags().StringVar(&cfg.DataplaneRuntime.BinaryPath, "binary-path", cfg.DataplaneRuntime.BinaryPath, "Binary path of Envoy executable") diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index 0ccd07a44fe9..d6496f4de343 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -14,8 +14,6 @@ import ( "github.com/pkg/errors" "github.com/sethvargo/go-retry" - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - kuma_dp "github.com/kumahq/kuma/pkg/config/app/kuma-dp" "github.com/kumahq/kuma/pkg/core" kuma_version "github.com/kumahq/kuma/pkg/version" @@ -118,14 +116,10 @@ func (b *remoteBootstrap) requestForBootstrap(url *net_url.URL, cfg kuma_dp.Conf if cfg.DataplaneRuntime.Token != "" { token = cfg.DataplaneRuntime.Token } - proxyType := mesh_proto.RegularDpType - if cfg.Dataplane.IsIngress { - proxyType = mesh_proto.IngressDpType - } request := types.BootstrapRequest{ Mesh: cfg.Dataplane.Mesh, Name: cfg.Dataplane.Name, - ProxyType: string(proxyType), + ProxyType: cfg.Dataplane.ProxyType, // if not set in config, the 0 will be sent which will result in providing default admin port // that is set in the control plane bootstrap params AdminPort: cfg.Dataplane.AdminPort.Lowest(), diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go index 99fe262ed24f..d10a7e1d697b 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go @@ -107,7 +107,7 @@ var _ = Describe("Remote Bootstrap", func() { { "mesh": "demo", "name": "sample", - "proxyType": "regular", + "proxyType": "dataplane", "adminPort": 4321, "dataplaneToken": "token", "dataplaneResource": "{\"type\":\"Dataplane\",\"mesh\":\"demo\",\"name\":\"sample\",\"creationTime\":\"0001-01-01T00:00:00Z\",\"modificationTime\":\"0001-01-01T00:00:00Z\"}", @@ -154,7 +154,7 @@ var _ = Describe("Remote Bootstrap", func() { { "mesh": "demo", "name": "sample", - "proxyType": "regular", + "proxyType": "dataplane", "adminPort": 4321, "dataplaneToken": "token", "dataplaneResource": "{\"type\":\"Dataplane\",\"mesh\":\"demo\",\"name\":\"sample\",\"creationTime\":\"0001-01-01T00:00:00Z\",\"modificationTime\":\"0001-01-01T00:00:00Z\"}", @@ -197,7 +197,7 @@ var _ = Describe("Remote Bootstrap", func() { { "mesh": "demo", "name": "sample", - "proxyType": "regular", + "proxyType": "dataplane", "dataplaneToken": "token", "dataplaneResource": "{\"type\":\"Dataplane\",\"mesh\":\"demo\",\"name\":\"sample\",\"creationTime\":\"0001-01-01T00:00:00Z\",\"modificationTime\":\"0001-01-01T00:00:00Z\"}", "version": { diff --git a/app/kumactl/cmd/generate/generate_dataplane_token.go b/app/kumactl/cmd/generate/generate_dataplane_token.go index dfbd1060ae1b..db4a7e1d922f 100644 --- a/app/kumactl/cmd/generate/generate_dataplane_token.go +++ b/app/kumactl/cmd/generate/generate_dataplane_token.go @@ -13,9 +13,9 @@ type generateDataplaneTokenContext struct { *kumactl_cmd.RootContext args struct { - name string - dpType string - tags map[string]string + name string + proxyType string + tags map[string]string } } @@ -49,7 +49,7 @@ $ kumactl generate dataplane-token --mesh demo --tag kuma.io/service=web,web-api tags[k] = strings.Split(v, ",") } name := ctx.args.name - token, err := client.Generate(name, pctx.Args.Mesh, tags, ctx.args.dpType) + token, err := client.Generate(name, pctx.Args.Mesh, tags, ctx.args.proxyType) if err != nil { return errors.Wrap(err, "failed to generate a dataplane token") } @@ -58,7 +58,9 @@ $ kumactl generate dataplane-token --mesh demo --tag kuma.io/service=web,web-api }, } cmd.Flags().StringVar(&ctx.args.name, "name", "", "name of the Dataplane") - cmd.Flags().StringVar(&ctx.args.dpType, "type", "", `type of the Dataplane ("dataplane", "ingress")`) + cmd.Flags().StringVar(&ctx.args.proxyType, "type", "", `type of the Dataplane ("dataplane", "ingress")`) + _ = cmd.Flags().MarkDeprecated("type", "please use --proxyType instead") + cmd.Flags().StringVar(&ctx.args.proxyType, "proxy-type", "", `type of the Dataplane ("dataplane", "ingress")`) cmd.Flags().StringToStringVar(&ctx.args.tags, "tag", nil, "required tag values for dataplane (split values by comma to provide multiple values)") return cmd } diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml index 0edcf5e25258..a3ad8f5452c3 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml @@ -779,7 +779,7 @@ spec: spec: serviceAccountName: kuma-control-plane nodeSelector: - + kubernetes.io/arch: amd64 kubernetes.io/os: linux containers: @@ -889,7 +889,7 @@ spec: spec: serviceAccountName: kuma-ingress nodeSelector: - + kubernetes.io/arch: amd64 kubernetes.io/os: linux containers: @@ -917,8 +917,8 @@ spec: value: 60s - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH value: /var/run/secrets/kubernetes.io/serviceaccount/token - - name: KUMA_DATAPLANE_IS_INGRESS - value: "true" + - name: KUMA_DATAPLANE_PROXY_TYPE + value: "ingress" args: - run - --log-level=info @@ -968,7 +968,7 @@ metadata: name: kuma-admission-mutating-webhook-configuration namespace: kuma-system labels: - + app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma webhooks: @@ -1017,8 +1017,8 @@ webhooks: - trafficpermissions - trafficroutes - traffictraces - - + + sideEffects: None - name: kuma-injector.kuma.io failurePolicy: Ignore @@ -1045,7 +1045,7 @@ metadata: name: kuma-validating-webhook-configuration namespace: kuma-system labels: - + app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma webhooks: @@ -1080,8 +1080,8 @@ webhooks: - trafficroutes - traffictraces - zones - - + + sideEffects: None - name: service.validator.kuma-admission.kuma.io failurePolicy: Ignore @@ -1124,4 +1124,4 @@ webhooks: - DELETE resources: - secrets - sideEffects: None \ No newline at end of file + sideEffects: None diff --git a/app/kumactl/pkg/tokens/client_test.go b/app/kumactl/pkg/tokens/client_test.go index 7002dc711954..a52304313fe7 100644 --- a/app/kumactl/pkg/tokens/client_test.go +++ b/app/kumactl/pkg/tokens/client_test.go @@ -37,7 +37,7 @@ type zoneIngressStaticTokenIssuer struct { var _ zoneingress.TokenIssuer = &zoneIngressStaticTokenIssuer{} func (z *zoneIngressStaticTokenIssuer) Generate(identity zoneingress.Identity) (zoneingress.Token, error) { - return fmt.Sprintf("token-for-%s", identity.Name), nil + return fmt.Sprintf("token-for-%s", identity.Zone), nil } func (z *zoneIngressStaticTokenIssuer) Validate(token zoneingress.Token) (zoneingress.Identity, error) { diff --git a/deployments/charts/kuma/README.md b/deployments/charts/kuma/README.md index 756a12a12ca5..d6b41af7c3d4 100644 --- a/deployments/charts/kuma/README.md +++ b/deployments/charts/kuma/README.md @@ -61,7 +61,6 @@ A Helm chart for the Kuma Control Plane | dataPlane.image.pullPolicy | string | `"IfNotPresent"` | Kuma DP ImagePullPolicy | | dataPlane.initImage.repository | string | `"kuma-init"` | The Kuma DP init image repository | | ingress.enabled | bool | `false` | If true, it deploys Ingress for cross cluster communication | -| ingress.mesh | string | `"default"` | Mesh to which Dataplane Ingress belongs to | | ingress.drainTime | string | `"30s"` | Time for which old listener will still be active as draining | | ingress.replicas | int | `1` | Number of replicas of the Ingress | | ingress.service.type | string | `"LoadBalancer"` | Service type of the Ingress | diff --git a/deployments/charts/kuma/templates/ingress-deployment.yaml b/deployments/charts/kuma/templates/ingress-deployment.yaml index 4cb848f25455..7992b1f06443 100644 --- a/deployments/charts/kuma/templates/ingress-deployment.yaml +++ b/deployments/charts/kuma/templates/ingress-deployment.yaml @@ -21,7 +21,6 @@ spec: metadata: annotations: kuma.io/ingress: enabled - kuma.io/mesh: {{ .Values.ingress.mesh }} {{- if .Values.ingress.annotations }} {{- range $key, $value := .Values.ingress.annotations }} {{ $key }}: {{ $value | quote }} @@ -61,8 +60,8 @@ spec: value: {{ .Values.ingress.drainTime }} - name: KUMA_DATAPLANE_RUNTIME_TOKEN_PATH value: /var/run/secrets/kubernetes.io/serviceaccount/token - - name: KUMA_DATAPLANE_IS_INGRESS - value: "true" + - name: KUMA_DATAPLANE_PROXY_TYPE + value: "ingress" args: - run - --log-level=info diff --git a/deployments/charts/kuma/values.yaml b/deployments/charts/kuma/values.yaml index b118afb26bf6..f37d784ae077 100644 --- a/deployments/charts/kuma/values.yaml +++ b/deployments/charts/kuma/values.yaml @@ -171,8 +171,6 @@ dataPlane: ingress: # -- If true, it deploys Ingress for cross cluster communication enabled: false - # -- Mesh to which Dataplane Ingress belongs to - mesh: default # -- Time for which old listener will still be active as draining drainTime: 30s # -- Number of replicas of the Ingress diff --git a/docs/cmd/kumactl/HELP.md b/docs/cmd/kumactl/HELP.md index 98a7978ab310..77f5ba5dbf5c 100644 --- a/docs/cmd/kumactl/HELP.md +++ b/docs/cmd/kumactl/HELP.md @@ -400,8 +400,8 @@ $ kumactl generate dataplane-token --mesh demo --tag kuma.io/service=web,web-api Flags: -h, --help help for dataplane-token --name string name of the Dataplane + --proxy-type string type of the Dataplane ("dataplane", "ingress") --tag stringToString required tag values for dataplane (split values by comma to provide multiple values) (default []) - --type string type of the Dataplane ("dataplane", "ingress") Global Flags: --config-file string path to the configuration file to use diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index a69c36ef2fd0..13a52aafc2a4 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -7,6 +7,8 @@ import ( "github.com/pkg/errors" "go.uber.org/multierr" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/config" config_types "github.com/kumahq/kuma/pkg/config/types" ) @@ -109,8 +111,8 @@ type Dataplane struct { Mesh string `yaml:"mesh,omitempty" envconfig:"kuma_dataplane_mesh"` // Dataplane name. Name string `yaml:"name,omitempty" envconfig:"kuma_dataplane_name"` - // IsIngress if true, runs kuma-dp as a zone ingress - IsIngress bool `yaml:"isIngress,omitempty" envconfig:"kuma_dataplane_is_ingress"` + // ProxyType defines mode which should be used, supported values: 'dataplane', 'ingress' + ProxyType string `yaml:"isIngress,omitempty" envconfig:"kuma_dataplane_proxy_type"` // Port (or range of ports to choose from) for Envoy Admin API to listen on. // Empty value indicates that Envoy Admin API should not be exposed over TCP. // Format: "9901 | 9901-9999 | 9901- | -9901". @@ -184,7 +186,11 @@ func (d *Dataplane) Sanitize() { } func (d *Dataplane) Validate() (errs error) { - if d.Mesh == "" && !d.IsIngress { + proxyType := mesh_proto.ProxyType(d.ProxyType) + if err := proxyType.IsValid(); err != nil { + errs = multierr.Append(errs, err) + } + if d.Mesh == "" && proxyType != mesh_proto.IngressProxyType { errs = multierr.Append(errs, errors.Errorf(".Mesh must be non-empty")) } if d.Name == "" { diff --git a/pkg/core/resources/apis/mesh/dataplane_overview_helpers.go b/pkg/core/resources/apis/mesh/dataplane_overview_helpers.go index 94504b476c37..15235181ce43 100644 --- a/pkg/core/resources/apis/mesh/dataplane_overview_helpers.go +++ b/pkg/core/resources/apis/mesh/dataplane_overview_helpers.go @@ -31,7 +31,7 @@ func (t *DataplaneOverviewResource) GetStatus() (Status, []string) { allInboundsOffline := len(errs) == len(t.Spec.Dataplane.Networking.Inbound) allInboundsOnline := len(errs) == 0 - if t.Spec.Dataplane.DpType() == mesh_proto.GatewayDpType { + if t.Spec.Dataplane.ProxyType() == mesh_proto.GatewayProxyType { allInboundsOffline = false allInboundsOnline = true } diff --git a/pkg/core/resources/apis/mesh/zone_ingress_helpers.go b/pkg/core/resources/apis/mesh/zone_ingress_helpers.go index 9dba4958849f..93db26ac298a 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_helpers.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_helpers.go @@ -12,10 +12,10 @@ func (r *ZoneIngressResource) UsesInboundInterface(address net.IP, port uint32) if r == nil { return false } - if port == r.Spec.Port && overlap(address, net.ParseIP(r.Spec.Address)) { + if port == r.Spec.GetNetworking().GetPort() && overlap(address, net.ParseIP(r.Spec.GetNetworking().GetAddress())) { return true } - if port == r.Spec.AdvertisedPort && overlap(address, net.ParseIP(r.Spec.AdvertisedAddress)) { + if port == r.Spec.GetNetworking().GetAdvertisedPort() && overlap(address, net.ParseIP(r.Spec.GetNetworking().GetAdvertisedAddress())) { return true } return false @@ -32,7 +32,7 @@ func (r *ZoneIngressResource) HasPublicAddress() bool { if r == nil { return false } - return r.Spec.GetAdvertisedAddress() != "" && r.Spec.GetAdvertisedPort() != 0 + return r.Spec.GetNetworking().GetAdvertisedAddress() != "" && r.Spec.GetNetworking().GetAdvertisedPort() != 0 } func NewZoneIngressResourceFromDataplane(dataplane *DataplaneResource) (*ZoneIngressResource, error) { @@ -62,8 +62,10 @@ func convert(dataplane *mesh_proto.Dataplane) (*mesh_proto.ZoneIngress, error) { }) } return &mesh_proto.ZoneIngress{ - Address: dataplane.GetNetworking().GetAddress(), - Port: dataplane.GetNetworking().Inbound[0].GetPort(), + Networking: &mesh_proto.ZoneIngress_Networking{ + Address: dataplane.GetNetworking().GetAddress(), + Port: dataplane.GetNetworking().Inbound[0].GetPort(), + }, AvailableServices: availableServices, }, nil } diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator.go b/pkg/core/resources/apis/mesh/zone_ingress_validator.go index f1bcbee35bea..d8b5ae5e0e55 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_validator.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator.go @@ -16,25 +16,25 @@ func validateZoneIngress(path validators.PathBuilder, ingress *mesh_proto.ZoneIn return validators.ValidationError{} } var err validators.ValidationError - if ingress.GetAdvertisedAddress() == "" && ingress.GetAdvertisedPort() != 0 { + if ingress.GetNetworking().GetAdvertisedAddress() == "" && ingress.GetNetworking().GetAdvertisedPort() != 0 { err.AddViolationAt(path.Field("advertisedAddress"), `has to be defined with advertisedPort`) } - if ingress.GetAdvertisedPort() == 0 && ingress.GetAdvertisedAddress() != "" { + if ingress.GetNetworking().GetAdvertisedPort() == 0 && ingress.GetNetworking().GetAdvertisedAddress() != "" { err.AddViolationAt(path.Field("advertisedPort"), `has to be defined with advertisedAddress`) } - if ingress.GetAddress() != "" { - err.Add(validateAddress(path.Field("address"), ingress.GetAddress())) + if ingress.GetNetworking().GetAddress() != "" { + err.Add(validateAddress(path.Field("address"), ingress.GetNetworking().GetAddress())) } - if ingress.GetAdvertisedAddress() != "" { - err.Add(validateAddress(path.Field("advertisedAddress"), ingress.GetAdvertisedAddress())) + if ingress.GetNetworking().GetAdvertisedAddress() != "" { + err.Add(validateAddress(path.Field("advertisedAddress"), ingress.GetNetworking().GetAdvertisedAddress())) } - if ingress.GetPort() == 0 { + if ingress.GetNetworking().GetPort() == 0 { err.AddViolationAt(path.Field("port"), `port has to be defined`) } - if ingress.GetPort() > 65535 { + if ingress.GetNetworking().GetPort() > 65535 { err.AddViolationAt(path.Field("port"), `port has to be in range of [1, 65535]`) } - if ingress.GetAdvertisedPort() > 65535 { + if ingress.GetNetworking().GetAdvertisedPort() > 65535 { err.AddViolationAt(path.Field("advertisedPort"), `port has to be in range of [1, 65535]`) } for i, ingressInterface := range ingress.GetAvailableServices() { diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go index 73f02e5e9f97..13d0440ab186 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go @@ -32,10 +32,11 @@ var _ = Describe("Dataplane", func() { Entry("with advertised address and port", ` type: ZoneIngress name: zi-1 - address: 192.168.0.1 - advertisedAddress: 10.0.0.1 - port: 10001 - advertisedPort: 1234 + networking: + address: 192.168.0.1 + advertisedAddress: 10.0.0.1 + port: 10001 + advertisedPort: 1234 availableServices: - tags: kuma.io/service: backend @@ -49,10 +50,11 @@ var _ = Describe("Dataplane", func() { Entry("with advertised ipv6 address and port", ` type: ZoneIngress name: zi-1 - address: 192.168.0.1 - advertisedAddress: ::ffff:0a00:0001 - port: 10001 - advertisedPort: 1234 + networking: + address: 192.168.0.1 + advertisedAddress: ::ffff:0a00:0001 + port: 10001 + advertisedPort: 1234 availableServices: - tags: kuma.io/service: backend @@ -67,8 +69,9 @@ var _ = Describe("Dataplane", func() { Entry("without advertised address and port", ` type: ZoneIngress name: zi-1 - address: 192.168.0.1 - port: 10001 + networking: + address: 192.168.0.1 + port: 10001 availableServices: []`, ), ) @@ -101,9 +104,10 @@ var _ = Describe("Dataplane", func() { dataplane: ` type: ZoneIngress name: zi-1 - address: 192.168.0.1 - advertisedAddress: 10.0.0.1 - advertisedPort: 1234 + networking: + address: 192.168.0.1 + advertisedAddress: 10.0.0.1 + advertisedPort: 1234 availableServices: - tags: kuma.io/service: backend @@ -122,10 +126,11 @@ var _ = Describe("Dataplane", func() { dataplane: ` type: ZoneIngress name: zi-1 - address: 192.168.0.1 - advertisedAddress: "!@#" - port: 10001 - advertisedPort: 100000 + networking: + address: 192.168.0.1 + advertisedAddress: "!@#" + port: 10001 + advertisedPort: 100000 availableServices: - tags: kuma.io/service: backend diff --git a/pkg/core/xds/metadata.go b/pkg/core/xds/metadata.go index b2e367099717..6ed76497c532 100644 --- a/pkg/core/xds/metadata.go +++ b/pkg/core/xds/metadata.go @@ -51,7 +51,7 @@ type DataplaneMetadata struct { DNSPort uint32 EmptyDNSPort uint32 DynamicMetadata map[string]string - ProxyType mesh_proto.DpType + ProxyType mesh_proto.ProxyType } func (m *DataplaneMetadata) GetDataplaneTokenPath() string { @@ -82,9 +82,9 @@ func (m *DataplaneMetadata) GetZoneIngressResource() *core_mesh.ZoneIngressResou return m.ZoneIngressResource } -func (m *DataplaneMetadata) GetProxyType() mesh_proto.DpType { +func (m *DataplaneMetadata) GetProxyType() mesh_proto.ProxyType { if m == nil || m.ProxyType == "" { - return mesh_proto.RegularDpType + return mesh_proto.DataplaneProxyType } return m.ProxyType } @@ -129,7 +129,7 @@ func DataplaneMetadataFromXdsMetadata(xdsMetadata *_struct.Struct) *DataplaneMet metadata.DataplaneToken = field.GetStringValue() } if field := xdsMetadata.Fields[fieldDataplaneProxyType]; field != nil { - metadata.ProxyType = mesh_proto.DpType(field.GetStringValue()) + metadata.ProxyType = mesh_proto.ProxyType(field.GetStringValue()) } metadata.AdminPort = uint32Metadata(xdsMetadata, fieldDataplaneAdminPort) metadata.DNSPort = uint32Metadata(xdsMetadata, fieldDataplaneDNSPort) diff --git a/pkg/dns/outbound_test.go b/pkg/dns/outbound_test.go index 3abfc222765d..0b2a4253d83e 100644 --- a/pkg/dns/outbound_test.go +++ b/pkg/dns/outbound_test.go @@ -278,6 +278,89 @@ var _ = Describe("VIPOutbounds", func() { port: 80 tags: kuma.io/service: third-external-service +` + Expect(proto.ToYAML(actual)).To(MatchYAML(expected)) + }) + + It("should take ingresses into account", func() { + dataplane := &core_mesh.DataplaneResource{ + Meta: &test_model.ResourceMeta{Name: "dp1", Mesh: "default"}, + Spec: &mesh_proto.Dataplane{ + Networking: &mesh_proto.Dataplane_Networking{ + Address: "192.168.0.1", + Inbound: []*mesh_proto.Dataplane_Networking_Inbound{{ + Port: 8080, + Tags: map[string]string{ + "kuma.io/service": "backend", + }}, + }, + }, + }, + } + + vipList := vips.List{ + "old-ingress-svc-1": "240.0.0.0", + "old-ingress-svc-2": "240.0.0.1", + "new-ingress-svc-1": "240.0.0.2", + "new-ingress-svc-2": "240.0.0.3", + } + + otherDataplanes := []*core_mesh.DataplaneResource{{ + Meta: &test_model.ResourceMeta{Name: "old-ingress", Mesh: "default"}, + Spec: &mesh_proto.Dataplane{ + Networking: &mesh_proto.Dataplane_Networking{ + Inbound: []*mesh_proto.Dataplane_Networking_Inbound{{ + Port: 10001, + Address: "192.168.0.2", + Tags: map[string]string{ + "kuma.io/service": "ingress", + }, + }}, + Ingress: &mesh_proto.Dataplane_Networking_Ingress{ + AvailableServices: []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{ + {Mesh: "default", Tags: map[string]string{mesh_proto.ServiceTag: "old-ingress-svc-1"}}, + {Mesh: "default", Tags: map[string]string{mesh_proto.ServiceTag: "old-ingress-svc-2"}}, + }, + }, + }, + }, + }} + + zoneIngresses := []*core_mesh.ZoneIngressResource{{ + Meta: &test_model.ResourceMeta{Name: "new-ingress", Mesh: model.NoMesh}, + Spec: &mesh_proto.ZoneIngress{ + Networking: &mesh_proto.ZoneIngress_Networking{ + Address: "192.168.0.3", + Port: 10001, + }, + AvailableServices: []*mesh_proto.ZoneIngress_AvailableService{ + {Mesh: "default", Tags: map[string]string{mesh_proto.ServiceTag: "new-ingress-svc-1"}}, + {Mesh: "default", Tags: map[string]string{mesh_proto.ServiceTag: "new-ingress-svc-2"}}, + }, + }, + }} + + actual := &mesh_proto.Dataplane_Networking{} + actual.Outbound = dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), otherDataplanes, zoneIngresses, vipList, nil) + + expected := ` + outbound: + - address: 240.0.0.2 + port: 80 + tags: + kuma.io/service: new-ingress-svc-1 + - address: 240.0.0.3 + port: 80 + tags: + kuma.io/service: new-ingress-svc-2 + - address: 240.0.0.0 + port: 80 + tags: + kuma.io/service: old-ingress-svc-1 + - address: 240.0.0.1 + port: 80 + tags: + kuma.io/service: old-ingress-svc-2 ` Expect(proto.ToYAML(actual)).To(MatchYAML(expected)) }) diff --git a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go index e4597e117652..e30b0607672d 100644 --- a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go @@ -34,8 +34,8 @@ func (p *PodConverter) IngressFor(zoneIngress *mesh_proto.ZoneIngress, pod *kube return errors.Errorf("generated %d inbound interfaces, expected 1. Interfaces: %v", len(ifaces), ifaces) } - zoneIngress.Address = pod.Status.PodIP - zoneIngress.Port = ifaces[0].Port + zoneIngress.Networking.Address = pod.Status.PodIP + zoneIngress.Networking.Port = ifaces[0].Port coords, err := p.coordinatesFromAnnotations(pod.Annotations) if err != nil { @@ -50,8 +50,8 @@ func (p *PodConverter) IngressFor(zoneIngress *mesh_proto.ZoneIngress, pod *kube } if coords != nil { - zoneIngress.AdvertisedAddress = coords.address - zoneIngress.AdvertisedPort = coords.port + zoneIngress.Networking.AdvertisedAddress = coords.address + zoneIngress.Networking.AdvertisedPort = coords.port } return nil } diff --git a/pkg/test/kds/samples/resources.go b/pkg/test/kds/samples/resources.go index 8a220e3b193e..99d194ad3805 100644 --- a/pkg/test/kds/samples/resources.go +++ b/pkg/test/kds/samples/resources.go @@ -98,10 +98,12 @@ var ( }, } ZoneIngress = &mesh_proto.ZoneIngress{ - Address: "127.0.0.1", - Port: 80, - AdvertisedAddress: "192.168.0.1", - AdvertisedPort: 10001, + Networking: &mesh_proto.ZoneIngress_Networking{ + Address: "127.0.0.1", + Port: 80, + AdvertisedAddress: "192.168.0.1", + AdvertisedPort: 10001, + }, AvailableServices: []*mesh_proto.ZoneIngress_AvailableService{{ Tags: map[string]string{ "service": "backend", diff --git a/pkg/tokens/builtin/issuer/issuer.go b/pkg/tokens/builtin/issuer/issuer.go index b6857134940d..8c087e076ba0 100644 --- a/pkg/tokens/builtin/issuer/issuer.go +++ b/pkg/tokens/builtin/issuer/issuer.go @@ -9,18 +9,18 @@ import ( type Token = string -type DpType = string - -const ( - DpTypeDataplane = "dataplane" - DpTypeIngress = "ingress" -) +//type DpType = string +// +//const ( +// DpTypeDataplane = "dataplane" +// DpTypeIngress = "ingress" +//) type DataplaneIdentity struct { Name string Mesh string Tags mesh_proto.MultiValueTagSet - Type DpType + Type mesh_proto.ProxyType } // DataplaneTokenIssuer issues Dataplane Tokens used then for proving identity of the dataplanes. @@ -77,7 +77,7 @@ func (i *jwtTokenIssuer) Generate(identity DataplaneIdentity) (Token, error) { Name: identity.Name, Mesh: identity.Mesh, Tags: tags, - Type: identity.Type, + Type: string(identity.Type), StandardClaims: jwt.StandardClaims{}, } @@ -111,7 +111,7 @@ func (i *jwtTokenIssuer) Validate(rawToken Token, meshName string) (DataplaneIde Mesh: c.Mesh, Name: c.Name, Tags: mesh_proto.MultiValueTagSetFrom(c.Tags), - Type: c.Type, + Type: mesh_proto.ProxyType(c.Type), } return id, nil } diff --git a/pkg/tokens/builtin/server/types/zoneingress_token_request.go b/pkg/tokens/builtin/server/types/zoneingress_token_request.go index 0b18f65cfcb6..0ea97ded97c4 100644 --- a/pkg/tokens/builtin/server/types/zoneingress_token_request.go +++ b/pkg/tokens/builtin/server/types/zoneingress_token_request.go @@ -1,5 +1,5 @@ package types type ZoneIngressTokenRequest struct { - Name string `json:"name"` + Zone string `json:"zone"` } diff --git a/pkg/tokens/builtin/server/webservice.go b/pkg/tokens/builtin/server/webservice.go index 6cb0df5a3d39..fa94204d2f44 100644 --- a/pkg/tokens/builtin/server/webservice.go +++ b/pkg/tokens/builtin/server/webservice.go @@ -59,7 +59,7 @@ func (d *tokenWebService) handleIdentityRequest(request *restful.Request, respon token, err := d.issuer.Generate(issuer.DataplaneIdentity{ Mesh: idReq.Mesh, Name: idReq.Name, - Type: idReq.Type, + Type: mesh_proto.ProxyType(idReq.Type), Tags: mesh_proto.MultiValueTagSetFrom(idReq.Tags), }) if err != nil { @@ -81,15 +81,8 @@ func (d *tokenWebService) handleZoneIngressIdentityRequest(request *restful.Requ return } - if idReq.Name == "" { - verr := validators.ValidationError{} - verr.AddViolation("name", "cannot be empty") - errors.HandleError(response, verr.OrNil(), "Invalid request") - return - } - token, err := d.zoneIngressIssuer.Generate(zoneingress.Identity{ - Name: idReq.Name, + Zone: idReq.Zone, }) if err != nil { errors.HandleError(response, err, "Could not issue a token") diff --git a/pkg/tokens/builtin/server/webservice_test.go b/pkg/tokens/builtin/server/webservice_test.go index 2ff6b57a5fe0..faacffc1db80 100644 --- a/pkg/tokens/builtin/server/webservice_test.go +++ b/pkg/tokens/builtin/server/webservice_test.go @@ -42,7 +42,7 @@ type zoneIngressStaticTokenIssuer struct { var _ zoneingress.TokenIssuer = &zoneIngressStaticTokenIssuer{} func (z *zoneIngressStaticTokenIssuer) Generate(identity zoneingress.Identity) (zoneingress.Token, error) { - return fmt.Sprintf("token-for-%s", identity.Name), nil + return fmt.Sprintf("token-for-%s", identity.Zone), nil } func (z *zoneIngressStaticTokenIssuer) Validate(token zoneingress.Token) (zoneingress.Identity, error) { diff --git a/pkg/tokens/builtin/zoneingress/issuer.go b/pkg/tokens/builtin/zoneingress/issuer.go index cd3ec196670d..d6ad9eab4c57 100644 --- a/pkg/tokens/builtin/zoneingress/issuer.go +++ b/pkg/tokens/builtin/zoneingress/issuer.go @@ -8,7 +8,7 @@ import ( type Token = string type Identity struct { - Name string + Zone string } // TokenIssuer issues Zone Ingress Tokens used then for proving identity of the zone ingresses. @@ -20,7 +20,7 @@ type TokenIssuer interface { } type claims struct { - Name string + Zone string jwt.StandardClaims } @@ -54,7 +54,7 @@ func (j *jwtTokenIssuer) Generate(identity Identity) (Token, error) { } c := claims{ - Name: identity.Name, + Zone: identity.Zone, StandardClaims: jwt.StandardClaims{}, } @@ -85,7 +85,7 @@ func (j *jwtTokenIssuer) Validate(rawToken Token) (Identity, error) { } id := Identity{ - Name: c.Name, + Zone: c.Zone, } return id, nil } diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index 310e500a8171..7c228553e0d7 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -120,10 +120,10 @@ func (a *authCallbacks) credential(streamID core_xds.StreamID) (Credential, erro func (a *authCallbacks) authenticate(credential Credential, req util_xds.DiscoveryRequest) error { md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) switch md.GetProxyType() { - case mesh_proto.IngressDpType: + case mesh_proto.IngressProxyType: return a.authenticateZoneIngress(credential, req) default: - return a.authenticateRegularDataplane(credential, req) + return a.authenticateDataplane(credential, req) } } @@ -157,7 +157,7 @@ func (a *authCallbacks) authenticateZoneIngress(credential Credential, req util_ return nil } -func (a *authCallbacks) authenticateRegularDataplane(credential Credential, req util_xds.DiscoveryRequest) error { +func (a *authCallbacks) authenticateDataplane(credential Credential, req util_xds.DiscoveryRequest) error { dataplane := core_mesh.NewDataplaneResource() md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) if md.GetDataplaneResource() != nil { diff --git a/pkg/xds/auth/callbacks_test.go b/pkg/xds/auth/callbacks_test.go index 31dcabc6b95f..a50d4fa794b9 100644 --- a/pkg/xds/auth/callbacks_test.go +++ b/pkg/xds/auth/callbacks_test.go @@ -13,7 +13,7 @@ import ( mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_manager "github.com/kumahq/kuma/pkg/core/resources/manager" - "github.com/kumahq/kuma/pkg/core/resources/model" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" "github.com/kumahq/kuma/pkg/core/resources/model/rest" core_store "github.com/kumahq/kuma/pkg/core/resources/store" "github.com/kumahq/kuma/pkg/plugins/resources/memory" @@ -26,7 +26,8 @@ import ( ) type testAuthenticator struct { - callCounter int + callCounter int + zoneCallCounter int } var _ auth.Authenticator = &testAuthenticator{} @@ -40,7 +41,11 @@ func (t *testAuthenticator) Authenticate(ctx context.Context, dataplane *core_me } func (t *testAuthenticator) AuthenticateZoneIngress(ctx context.Context, zoneIngress *core_mesh.ZoneIngressResource, credential auth.Credential) error { - return nil + t.zoneCallCounter++ + if credential == "zone pass" { + return nil + } + return errors.New("invalid credential") } var _ = Describe("Auth Callbacks", func() { @@ -71,16 +76,31 @@ var _ = Describe("Auth Callbacks", func() { }, } + zoneIngress := &core_mesh.ZoneIngressResource{ + Meta: &test_model.ResourceMeta{ + Name: "ingress", + Mesh: core_model.NoMesh, + }, + Spec: &mesh_proto.ZoneIngress{ + Networking: &mesh_proto.ZoneIngress_Networking{ + Address: "1.1.1.1", + Port: 10001, + }, + }, + } + BeforeEach(func() { memStore := memory.NewStore() resManager = core_manager.NewResourceManager(memStore) testAuth = &testAuthenticator{} callbacks = util_xds_v2.AdaptCallbacks(auth.NewCallbacks(resManager, testAuth, auth.DPNotFoundRetry{})) - err := resManager.Create(context.Background(), core_mesh.NewMeshResource(), core_store.CreateByKey(model.DefaultMesh, model.NoMesh)) + err := resManager.Create(context.Background(), core_mesh.NewMeshResource(), core_store.CreateByKey(core_model.DefaultMesh, core_model.NoMesh)) Expect(err).ToNot(HaveOccurred()) err = resManager.Create(context.Background(), dpRes, core_store.CreateByKey("web-01", "default")) Expect(err).ToNot(HaveOccurred()) + err = resManager.Create(context.Background(), zoneIngress, core_store.CreateBy(core_model.MetaToResourceKey(zoneIngress.GetMeta()))) + Expect(err).ToNot(HaveOccurred()) }) It("should authenticate only first request on the stream", func() { @@ -223,4 +243,43 @@ var _ = Describe("Auth Callbacks", func() { // then Expect(err).To(MatchError("authentication failed: invalid credential")) }) + + It("should authenticate ingress", func() { + // given + ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{"authorization": "zone pass"})) + streamID := int64(1) + + // when + err := callbacks.OnStreamOpen(ctx, streamID, "") + + // then + Expect(err).ToNot(HaveOccurred()) + + // when + err = callbacks.OnStreamRequest(streamID, &envoy_api.DiscoveryRequest{ + Node: &envoy_core.Node{ + Id: ".ingress", + Metadata: &pstruct.Struct{ + Fields: map[string]*pstruct.Value{ + "dataplane.proxyType": { + Kind: &pstruct.Value_StringValue{ + StringValue: "ingress", + }, + }, + }, + }, + }, + }) + + // then + Expect(err).ToNot(HaveOccurred()) + Expect(testAuth.zoneCallCounter).To(Equal(1)) + + // when send second request that is already authenticated + err = callbacks.OnStreamRequest(streamID, &envoy_api.DiscoveryRequest{}) + + // then auth is called only once + Expect(err).ToNot(HaveOccurred()) + Expect(testAuth.zoneCallCounter).To(Equal(1)) + }) }) diff --git a/pkg/xds/auth/components/components.go b/pkg/xds/auth/components/components.go index b3a5ffd4000a..c699b6caada3 100644 --- a/pkg/xds/auth/components/components.go +++ b/pkg/xds/auth/components/components.go @@ -33,7 +33,7 @@ func NewUniversalAuthenticator(rt core_runtime.Runtime) (auth.Authenticator, err if err != nil { return nil, err } - return universal_auth.NewAuthenticator(issuer, zoneIngressIssuer), nil + return universal_auth.NewAuthenticator(issuer, zoneIngressIssuer, rt.Config().Multizone.Zone.Name), nil } func DefaultAuthenticator(rt core_runtime.Runtime) (auth.Authenticator, error) { diff --git a/pkg/xds/auth/universal/auth_test.go b/pkg/xds/auth/universal/auth_test.go index 42a091a911cd..251120367c85 100644 --- a/pkg/xds/auth/universal/auth_test.go +++ b/pkg/xds/auth/universal/auth_test.go @@ -84,7 +84,7 @@ var _ = Describe("Authentication flow", func() { BeforeEach(func() { resStore = memory.NewStore() - authenticator = universal.NewAuthenticator(issuer, zoneIngressIssuer) + authenticator = universal.NewAuthenticator(issuer, zoneIngressIssuer, "zone-1") err := resStore.Create(context.Background(), &dpRes, core_store.CreateByKey("dp-1", "default")) Expect(err).ToNot(HaveOccurred()) @@ -136,7 +136,7 @@ var _ = Describe("Authentication flow", func() { }), Entry("should auth with ingress token", testCase{ id: builtin_issuer.DataplaneIdentity{ - Type: builtin_issuer.DpTypeIngress, + Type: mesh_proto.IngressProxyType, }, dpRes: &ingressDp, }), @@ -209,14 +209,14 @@ var _ = Describe("Authentication flow", func() { }), Entry("regular dataplane and ingress type", testCase{ id: builtin_issuer.DataplaneIdentity{ - Type: builtin_issuer.DpTypeIngress, + Type: mesh_proto.IngressProxyType, }, dpRes: &dpRes, err: `dataplane is of type Dataplane but token allows only for the "ingress" type`, }), Entry("ingress dataplane and dataplane type", testCase{ id: builtin_issuer.DataplaneIdentity{ - Type: builtin_issuer.DpTypeDataplane, + Type: mesh_proto.DataplaneProxyType, }, dpRes: &ingressDp, err: `dataplane is of type Ingress but token allows only for the "dataplane" type`, diff --git a/pkg/xds/auth/universal/authenticator.go b/pkg/xds/auth/universal/authenticator.go index 22aef9b28be6..25891a6544fd 100644 --- a/pkg/xds/auth/universal/authenticator.go +++ b/pkg/xds/auth/universal/authenticator.go @@ -13,10 +13,11 @@ import ( "github.com/kumahq/kuma/pkg/xds/auth" ) -func NewAuthenticator(issuer builtin_issuer.DataplaneTokenIssuer, zoneIngressIssuer zoneingress.TokenIssuer) auth.Authenticator { +func NewAuthenticator(issuer builtin_issuer.DataplaneTokenIssuer, zoneIngressIssuer zoneingress.TokenIssuer, zone string) auth.Authenticator { return &universalAuthenticator{ issuer: issuer, zoneIngressIssuer: zoneIngressIssuer, + zone: zone, } } @@ -32,6 +33,7 @@ func NewAuthenticator(issuer builtin_issuer.DataplaneTokenIssuer, zoneIngressIss type universalAuthenticator struct { issuer builtin_issuer.DataplaneTokenIssuer zoneIngressIssuer zoneingress.TokenIssuer + zone string } func (u *universalAuthenticator) Authenticate(ctx context.Context, dataplane *core_mesh.DataplaneResource, credential auth.Credential) error { @@ -60,22 +62,22 @@ func (u *universalAuthenticator) AuthenticateZoneIngress(ctx context.Context, zo if err != nil { return err } - if zoneIngress.Meta.GetName() != identity.Name { - return errors.Errorf("zone ingress name from requestor: %s is different than in token: %s", zoneIngress.Meta.GetName(), identity.Name) + if u.zone != identity.Zone { + return errors.Errorf("zone ingress zone from requestor: %s is different than in token: %s", u.zone, identity.Zone) } return nil } -func validateType(dataplane *core_mesh.DataplaneResource, dpType builtin_issuer.DpType) error { - if dpType == "" { // if dp type is not explicitly specified we assume it's dataplane so we force Ingress token - dpType = builtin_issuer.DpTypeDataplane +func validateType(dataplane *core_mesh.DataplaneResource, proxyType mesh_proto.ProxyType) error { + if proxyType == "" { // if dp type is not explicitly specified we assume it's dataplane so we force Ingress token + proxyType = mesh_proto.DataplaneProxyType } - if dataplane.Spec.IsIngress() && dpType != builtin_issuer.DpTypeIngress { - return errors.Errorf("dataplane is of type Ingress but token allows only for the %q type", dpType) + if dataplane.Spec.IsIngress() && proxyType != mesh_proto.IngressProxyType { + return errors.Errorf("dataplane is of type Ingress but token allows only for the %q type", proxyType) } - if !dataplane.Spec.IsIngress() && dpType == builtin_issuer.DpTypeIngress { - return errors.Errorf("dataplane is of type Dataplane but token allows only for the %q type", dpType) + if !dataplane.Spec.IsIngress() && proxyType == mesh_proto.IngressProxyType { + return errors.Errorf("dataplane is of type Dataplane but token allows only for the %q type", proxyType) } return nil } diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index ae201b4e5be3..7e0508649516 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -78,13 +78,13 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr return nil, "", err } - proxyType := mesh_proto.DpType(request.ProxyType) + proxyType := mesh_proto.ProxyType(request.ProxyType) if request.ProxyType == "" { - proxyType = mesh_proto.RegularDpType + proxyType = mesh_proto.DataplaneProxyType } switch proxyType { - case mesh_proto.IngressDpType: + case mesh_proto.IngressProxyType: proxyId := core_xds.BuildProxyId(request.Mesh, request.Name) zoneIngress, err := b.zoneIngressFor(ctx, request, proxyId) if err != nil { @@ -96,7 +96,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr } b.hdsEnabled = false return b.generateFor(*proxyId, request, "ingress", adminPort) - case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: + case mesh_proto.DataplaneProxyType, mesh_proto.GatewayProxyType: proxyId := core_xds.BuildProxyId(request.Mesh, request.Name) dataplane, err := b.dataplaneFor(ctx, request, proxyId) if err != nil { diff --git a/pkg/xds/cache/mesh/snapshot.go b/pkg/xds/cache/mesh/snapshot.go index 79eac29544e1..18f4aab8f8ea 100644 --- a/pkg/xds/cache/mesh/snapshot.go +++ b/pkg/xds/cache/mesh/snapshot.go @@ -132,8 +132,8 @@ func (m *meshSnapshot) hashResource(r core_model.Resource) string { v.GetMeta().GetMesh(), v.GetMeta().GetName(), v.GetMeta().GetVersion(), - m.hashResolvedIPs(v.Spec.GetAddress()), - m.hashResolvedIPs(v.Spec.GetAdvertisedAddress()), + m.hashResolvedIPs(v.Spec.GetNetworking().GetAddress()), + m.hashResolvedIPs(v.Spec.GetNetworking().GetAdvertisedAddress()), }, ":") default: return strings.Join( diff --git a/pkg/xds/generator/admin_proxy_generator.go b/pkg/xds/generator/admin_proxy_generator.go index 9b688570950c..67e40c6dd286 100644 --- a/pkg/xds/generator/admin_proxy_generator.go +++ b/pkg/xds/generator/admin_proxy_generator.go @@ -113,5 +113,5 @@ func (g AdminProxyGenerator) getAddress(proxy *core_xds.Proxy) string { if proxy.Dataplane != nil { return proxy.Dataplane.Spec.GetNetworking().Address } - return proxy.ZoneIngress.Spec.Address + return proxy.ZoneIngress.Spec.GetNetworking().GetAddress() } diff --git a/pkg/xds/generator/ingress_generator.go b/pkg/xds/generator/ingress_generator.go index 5b6079f48c98..5fcff7261d81 100644 --- a/pkg/xds/generator/ingress_generator.go +++ b/pkg/xds/generator/ingress_generator.go @@ -69,9 +69,9 @@ func (i IngressGenerator) generateLDS( destinationsPerService map[string][]envoy_common.Tags, apiVersion envoy_common.APIVersion, ) (envoy_common.NamedResource, error) { - inboundListenerName := envoy_names.GetInboundListenerName(proxy.ZoneIngress.Spec.GetAddress(), proxy.ZoneIngress.Spec.GetPort()) + inboundListenerName := envoy_names.GetInboundListenerName(proxy.ZoneIngress.Spec.GetNetworking().GetAddress(), proxy.ZoneIngress.Spec.GetNetworking().GetPort()) inboundListenerBuilder := envoy_listeners.NewListenerBuilder(apiVersion). - Configure(envoy_listeners.InboundListener(inboundListenerName, ingress.Spec.GetAddress(), ingress.Spec.GetPort(), model.SocketAddressProtocolTCP)). + Configure(envoy_listeners.InboundListener(inboundListenerName, ingress.Spec.GetNetworking().GetAddress(), ingress.Spec.GetNetworking().GetPort(), model.SocketAddressProtocolTCP)). Configure(envoy_listeners.TLSInspector()) if len(proxy.ZoneIngress.Spec.AvailableServices) == 0 { diff --git a/pkg/xds/server/callbacks/dataplane_lifecycle.go b/pkg/xds/server/callbacks/dataplane_lifecycle.go index 494810586d57..e1c253d89ddb 100644 --- a/pkg/xds/server/callbacks/dataplane_lifecycle.go +++ b/pkg/xds/server/callbacks/dataplane_lifecycle.go @@ -34,7 +34,7 @@ type DataplaneLifecycle struct { // createdDpForStream stores map from StreamID to created ResourceKey of Dataplane. // we store nil values for streams without Dataplane in metadata to avoid accessing metadata with every DiscoveryRequest createdDpForStream map[xds.StreamID]*model.ResourceKey - proxyTypeForStream map[xds.StreamID]mesh_proto.DpType + proxyTypeForStream map[xds.StreamID]mesh_proto.ProxyType sync.RWMutex // protects createdDpForStream shutdownCh <-chan struct{} } @@ -45,7 +45,7 @@ func NewDataplaneLifecycle(resManager manager.ResourceManager, shutdownCh <-chan return &DataplaneLifecycle{ resManager: resManager, createdDpForStream: map[xds.StreamID]*model.ResourceKey{}, - proxyTypeForStream: map[xds.StreamID]mesh_proto.DpType{}, + proxyTypeForStream: map[xds.StreamID]mesh_proto.ProxyType{}, shutdownCh: shutdownCh, } } @@ -73,7 +73,7 @@ func (d *DataplaneLifecycle) OnStreamClosed(streamID int64) { return } - if proxyType == mesh_proto.RegularDpType { + if proxyType == mesh_proto.DataplaneProxyType { lifecycleLog.Info("unregistering dataplane", "dataplaneKey", key, "streamID", streamID) if err := d.unregisterDataplane(*key); err != nil { lifecycleLog.Error(err, "could not unregister dataplane") @@ -81,7 +81,7 @@ func (d *DataplaneLifecycle) OnStreamClosed(streamID int64) { return } - if proxyType == mesh_proto.IngressDpType { + if proxyType == mesh_proto.IngressProxyType { lifecycleLog.Info("unregistering zone ingress", "zoneIngressKey", key, "streamID", streamID) if err := d.unregisterZoneIngress(*key); err != nil { lifecycleLog.Error(err, "could not unregister zone ingress") @@ -104,25 +104,25 @@ func (d *DataplaneLifecycle) OnStreamRequest(streamID int64, request util_xds.Di md := xds.DataplaneMetadataFromXdsMetadata(request.Metadata()) - if md.GetProxyType() == mesh_proto.RegularDpType && md.GetDataplaneResource() != nil { + if md.GetProxyType() == mesh_proto.DataplaneProxyType && md.GetDataplaneResource() != nil { lifecycleLog.Info("registering dataplane", "dataplane", md.GetDataplaneResource(), "streamID", streamID, "nodeID", request.NodeId()) if err := d.registerDataplane(md.GetDataplaneResource()); err != nil { return errors.Wrap(err, "could not register dataplane passed in kuma-dp run") } key := model.MetaToResourceKey(md.GetDataplaneResource().GetMeta()) d.createdDpForStream[streamID] = &key - d.proxyTypeForStream[streamID] = mesh_proto.RegularDpType + d.proxyTypeForStream[streamID] = mesh_proto.DataplaneProxyType return nil } - if md.GetProxyType() == mesh_proto.IngressDpType && md.ZoneIngressResource != nil { + if md.GetProxyType() == mesh_proto.IngressProxyType && md.ZoneIngressResource != nil { lifecycleLog.Info("registering zone ingress", "zoneIngress", md.ZoneIngressResource, "streamID", streamID, "nodeID", request.NodeId()) if err := d.registerZoneIngress(md.ZoneIngressResource); err != nil { return errors.Wrap(err, "could not register zone ingress passed in kuma-dp run") } key := model.MetaToResourceKey(md.ZoneIngressResource.GetMeta()) d.createdDpForStream[streamID] = &key - d.proxyTypeForStream[streamID] = mesh_proto.IngressDpType + d.proxyTypeForStream[streamID] = mesh_proto.IngressProxyType return nil } diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index 5a7a3380f799..ba3a7b202e92 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -124,7 +124,7 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di statusTrackerLog.Error(err, "failed to extract version out of the Envoy metadata", "streamid", streamID, "metadata", req.Metadata()) } // kick off async Dataplane status flusher - if md.GetProxyType() != mesh_proto.IngressDpType { + if md.GetProxyType() != mesh_proto.IngressProxyType { go c.createStatusSink(state).Start(state.stop) } } else { diff --git a/pkg/xds/sync/dataplane_watchdog.go b/pkg/xds/sync/dataplane_watchdog.go index 432a0f1e5ef9..72a7c7a8e272 100644 --- a/pkg/xds/sync/dataplane_watchdog.go +++ b/pkg/xds/sync/dataplane_watchdog.go @@ -35,7 +35,7 @@ type DataplaneWatchdog struct { // state of watchdog lastHash string // last Mesh hash that was used to **successfully** generate Reconcile Envoy config - dpType mesh_proto.DpType + dpType mesh_proto.ProxyType proxyTypeSettled bool } @@ -56,20 +56,20 @@ func (d *DataplaneWatchdog) Sync() error { d.dpType = d.metadataTracker.Metadata(d.streamId).GetProxyType() } // backwards compatibility - if d.dpType == mesh_proto.RegularDpType && !d.proxyTypeSettled { + if d.dpType == mesh_proto.DataplaneProxyType && !d.proxyTypeSettled { dataplane := mesh_core.NewDataplaneResource() if err := d.dataplaneProxyBuilder.CachingResManager.Get(ctx, dataplane, store.GetBy(d.key)); err != nil { return err } if dataplane.Spec.IsIngress() { - d.dpType = mesh_proto.IngressDpType + d.dpType = mesh_proto.IngressProxyType } d.proxyTypeSettled = true } switch d.dpType { - case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: + case mesh_proto.DataplaneProxyType, mesh_proto.GatewayProxyType: return d.syncDataplane() - case mesh_proto.IngressDpType: + case mesh_proto.IngressProxyType: return d.syncIngress() default: // It might be a case that dp type is not yet inferred because there is no Dataplane definition yet. @@ -80,9 +80,9 @@ func (d *DataplaneWatchdog) Sync() error { func (d *DataplaneWatchdog) Cleanup() error { proxyID := core_xds.FromResourceKey(d.key) switch d.dpType { - case mesh_proto.RegularDpType, mesh_proto.GatewayDpType: + case mesh_proto.DataplaneProxyType, mesh_proto.GatewayProxyType: return d.dataplaneReconciler.Clear(&proxyID) - case mesh_proto.IngressDpType: + case mesh_proto.IngressProxyType: return d.ingressReconciler.Clear(&proxyID) default: return nil diff --git a/pkg/xds/topology/dataplanes.go b/pkg/xds/topology/dataplanes.go index be0f19a5f95b..9ee27c8fc834 100644 --- a/pkg/xds/topology/dataplanes.go +++ b/pkg/xds/topology/dataplanes.go @@ -88,19 +88,19 @@ func ResolveIngressPublicAddress(lookupIPFunc lookup.LookupIPFunc, dataplane *co } func ResolveZoneIngressPublicAddress(lookupIPFunc lookup.LookupIPFunc, zoneIngress *core_mesh.ZoneIngressResource) (*core_mesh.ZoneIngressResource, error) { - if zoneIngress.Spec.GetAdvertisedAddress() == "" { // Ingress may not have public address yet. + if zoneIngress.Spec.GetNetworking().GetAdvertisedAddress() == "" { // Ingress may not have public address yet. return zoneIngress, nil } - ips, err := lookupIPFunc(zoneIngress.Spec.GetAdvertisedAddress()) + ips, err := lookupIPFunc(zoneIngress.Spec.GetNetworking().GetAdvertisedAddress()) if err != nil { return nil, err } if len(ips) == 0 { - return nil, errors.Errorf("can't resolve address %v", zoneIngress.Spec.GetAdvertisedAddress()) + return nil, errors.Errorf("can't resolve address %v", zoneIngress.Spec.GetNetworking().GetAdvertisedAddress()) } - if zoneIngress.Spec.GetAdvertisedAddress() != ips[0].String() { // only if we resolve any address, in most cases this is IP not a hostname + if zoneIngress.Spec.GetNetworking().GetAdvertisedAddress() != ips[0].String() { // only if we resolve any address, in most cases this is IP not a hostname ziSpec := proto.Clone(zoneIngress.Spec).(*mesh_proto.ZoneIngress) - ziSpec.AdvertisedAddress = ips[0].String() + ziSpec.Networking.AdvertisedAddress = ips[0].String() return &core_mesh.ZoneIngressResource{ Meta: zoneIngress.Meta, Spec: ziSpec, diff --git a/pkg/xds/topology/outbound.go b/pkg/xds/topology/outbound.go index 31dbff00240c..85b2cab77f63 100644 --- a/pkg/xds/topology/outbound.go +++ b/pkg/xds/topology/outbound.go @@ -114,7 +114,7 @@ func fillIngressOutbounds( if !zi.HasPublicAddress() { continue // Zone Ingress is not reachable yet from other clusters. This may happen when Ingress Service is pending waiting on External IP on Kubernetes. } - ingressCoordinates := net.JoinHostPort(zi.Spec.GetAdvertisedAddress(), strconv.FormatUint(uint64(zi.Spec.GetAdvertisedPort()), 10)) + ingressCoordinates := net.JoinHostPort(zi.Spec.GetNetworking().GetAdvertisedAddress(), strconv.FormatUint(uint64(zi.Spec.GetNetworking().GetAdvertisedPort()), 10)) if ingressInstances[ingressCoordinates] { continue // many Ingress instances can be placed in front of one load balancer (all instances can have the same public address and port). In this case we only need one Instance avoiding creating unnecessary duplicated endpoints } @@ -124,8 +124,8 @@ func fillIngressOutbounds( } serviceName := service.Tags[mesh_proto.ServiceTag] outbound[serviceName] = append(outbound[serviceName], core_xds.Endpoint{ - Target: zi.Spec.GetAdvertisedAddress(), - Port: zi.Spec.GetAdvertisedPort(), + Target: zi.Spec.GetNetworking().GetAdvertisedAddress(), + Port: zi.Spec.GetNetworking().GetAdvertisedPort(), Tags: service.Tags, Weight: service.Instances, Locality: localityFromTags(mesh, priorityRemote, service.Tags), diff --git a/test/e2e/deploy/kuma_deploy_universal.go b/test/e2e/deploy/kuma_deploy_universal.go index 9798fc9d6c82..7e84a87bb904 100644 --- a/test/e2e/deploy/kuma_deploy_universal.go +++ b/test/e2e/deploy/kuma_deploy_universal.go @@ -66,8 +66,6 @@ name: %s Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken(nonDefaultMesh, "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") - Expect(err).ToNot(HaveOccurred()) // TODO: right now these tests are deliberately run WithHDS(false) // even if HDS is enabled without any ServiceProbes it still affects @@ -78,12 +76,14 @@ name: %s optsZone1 = append(optsZone1, WithGlobalAddress(globalCP.GetKDSServerAddress()), WithHDS(false)) + ingressTokenKuma3, err := globalCP.GenerateZoneIngressToken(Kuma3) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(core.Zone, optsZone1...)). Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal1", echoServerToken, WithTransparentProxy(true))). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma3)). Setup(zone1) Expect(err).ToNot(HaveOccurred()) err = zone1.VerifyKuma() @@ -94,12 +94,14 @@ name: %s optsZone2 = append(optsZone2, WithGlobalAddress(globalCP.GetKDSServerAddress()), WithHDS(false)) + ingressTokenKuma4, err := globalCP.GenerateZoneIngressToken(Kuma4) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(core.Zone, optsZone2...)). Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal2", echoServerToken, WithTransparentProxy(true))). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma4)). Setup(zone2) Expect(err).ToNot(HaveOccurred()) err = zone2.VerifyKuma() diff --git a/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go b/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go index 1e561bc8da2e..012afb6bce01 100644 --- a/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go +++ b/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go @@ -75,19 +75,20 @@ metadata: echoServerToken, err := globalK8s.GetKuma().GenerateDpToken("default", "echo-server_kuma-test_svc_8080") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalK8s.GetKuma().GenerateZoneIngressToken("ingress") - Expect(err).ToNot(HaveOccurred()) optsZoneUniversal = append(optsZoneUniversal, WithGlobalAddress(globalK8s.GetKuma().GetKDSServerAddress())) zoneUniversal = universalClusters.GetCluster(Kuma3) + ingressTokenKuma3, err := globalK8s.GetKuma().GenerateZoneIngressToken(Kuma3) + Expect(err).ToNot(HaveOccurred()) + err = NewClusterSetup(). Install(Kuma(core.Zone, optsZoneUniversal...)). Install(EchoServerUniversal("dp-echo-1", "default", "echo-universal-1", echoServerToken, WithProtocol("tcp"))). Install(EchoServerUniversal("dp-echo-2", "default", "echo-universal-2", echoServerToken, WithProtocol("tcp"), ProxyOnly(), ServiceProbe())). Install(EchoServerUniversal("dp-echo-3", "default", "echo-universal-3", echoServerToken, WithProtocol("tcp"))). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma3)). Setup(zoneUniversal) Expect(err).ToNot(HaveOccurred()) err = zoneUniversal.VerifyKuma() diff --git a/test/e2e/hybrid/kuma_hybrid.go b/test/e2e/hybrid/kuma_hybrid.go index 3902c189575d..1ad90226f9a8 100644 --- a/test/e2e/hybrid/kuma_hybrid.go +++ b/test/e2e/hybrid/kuma_hybrid.go @@ -72,8 +72,6 @@ metadata: Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken(nonDefaultMesh, "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") - Expect(err).ToNot(HaveOccurred()) // K8s Cluster 1 zone1 = k8sClusters.GetCluster(Kuma1) @@ -114,12 +112,14 @@ metadata: zone3 = universalClusters.GetCluster(Kuma3) optsZone3 = append(optsZone3, WithGlobalAddress(globalCP.GetKDSServerAddress())) + ingressTokenKuma3, err := globalCP.GenerateZoneIngressToken(Kuma3) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(core.Zone, optsZone3...)). Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal", echoServerToken, WithTransparentProxy(true))). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma3)). Setup(zone3) Expect(err).ToNot(HaveOccurred()) err = zone3.VerifyKuma() @@ -129,11 +129,13 @@ metadata: zone4 = universalClusters.GetCluster(Kuma4) optsZone4 = append(optsZone4, WithGlobalAddress(globalCP.GetKDSServerAddress())) + ingressTokenKuma4, err := globalCP.GenerateZoneIngressToken(Kuma4) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(core.Zone, optsZone4...)). Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken)). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma4)). Setup(zone4) Expect(err).ToNot(HaveOccurred()) err = zone4.VerifyKuma() diff --git a/test/e2e/hybrid/kuma_hybrid_kube_global.go b/test/e2e/hybrid/kuma_hybrid_kube_global.go index af85ff7390a8..1e8c07b37a98 100644 --- a/test/e2e/hybrid/kuma_hybrid_kube_global.go +++ b/test/e2e/hybrid/kuma_hybrid_kube_global.go @@ -42,19 +42,19 @@ func KubernetesUniversalDeploymentWhenGlobalIsOnK8S() { Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken("default", "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") - Expect(err).ToNot(HaveOccurred()) // Zone zoneCluster = universalClusters.GetCluster(Kuma3) optsZone = append(optsZone, WithGlobalAddress(globalCP.GetKDSServerAddress())) + ingressTokenKuma3, err := globalCP.GenerateZoneIngressToken(Kuma3) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(core.Zone, optsZone...)). Install(EchoServerUniversal(AppModeEchoServer, "default", "universal", echoServerToken)). Install(DemoClientUniversal(AppModeDemoClient, "default", demoClientToken)). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma3)). Setup(zoneCluster) Expect(err).ToNot(HaveOccurred()) err = zoneCluster.VerifyKuma() diff --git a/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go b/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go index 0f638ba6226a..6394ba8eb61a 100644 --- a/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go +++ b/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go @@ -69,18 +69,18 @@ spec: echoServerToken, err := globalCP.GenerateDpToken("default", "echo-server_kuma-test_svc_8080") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") - Expect(err).ToNot(HaveOccurred()) // Zone universal zoneUniversal = universalClusters.GetCluster(Kuma3) optsZoneUniversal = append(optsZoneUniversal, WithGlobalAddress(globalCP.GetKDSServerAddress())) + ingressTokenKuma3, err := globalCP.GenerateZoneIngressToken(Kuma3) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(config_core.Zone, optsZoneUniversal...)). Install(EchoServerUniversal(AppModeEchoServer, "default", "universal", echoServerToken)). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma3)). Setup(zoneUniversal) Expect(err).ToNot(HaveOccurred()) err = zoneUniversal.VerifyKuma() diff --git a/test/e2e/trafficroute/universal_multizone/traffic_route.go b/test/e2e/trafficroute/universal_multizone/traffic_route.go index 36026cea3059..e3fc980681fd 100644 --- a/test/e2e/trafficroute/universal_multizone/traffic_route.go +++ b/test/e2e/trafficroute/universal_multizone/traffic_route.go @@ -57,19 +57,19 @@ routing: Expect(err).ToNot(HaveOccurred()) demoClientToken, err := globalCP.GenerateDpToken(defaultMesh, "demo-client") Expect(err).ToNot(HaveOccurred()) - ingressToken, err := globalCP.GenerateZoneIngressToken("ingress") - Expect(err).ToNot(HaveOccurred()) // Cluster 1 zone1 = clusters.GetCluster(Kuma3) optsZone1 = []DeployOptionsFunc{ WithGlobalAddress(globalCP.GetKDSServerAddress()), } + ingressTokenKuma3, err := globalCP.GenerateZoneIngressToken(Kuma3) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(core.Zone, optsZone1...)). Install(DemoClientUniversal(AppModeDemoClient, defaultMesh, demoClientToken, WithTransparentProxy(true))). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma3)). Setup(zone1) Expect(err).ToNot(HaveOccurred()) err = zone1.VerifyKuma() @@ -80,6 +80,8 @@ routing: optsZone2 = []DeployOptionsFunc{ WithGlobalAddress(globalCP.GetKDSServerAddress()), } + ingressTokenKuma4, err := globalCP.GenerateZoneIngressToken(Kuma4) + Expect(err).ToNot(HaveOccurred()) err = NewClusterSetup(). Install(Kuma(core.Zone, optsZone2...)). @@ -113,7 +115,7 @@ routing: WithServiceName("another-test-server"), WithTransparentProxy(true), )). - Install(IngressUniversal(ingressToken)). + Install(IngressUniversal(ingressTokenKuma4)). Setup(zone2) Expect(err).ToNot(HaveOccurred()) err = zone2.VerifyKuma() diff --git a/test/framework/interface.go b/test/framework/interface.go index e317807f16c1..b9c06fb61218 100644 --- a/test/framework/interface.go +++ b/test/framework/interface.go @@ -315,5 +315,5 @@ type ControlPlane interface { GetKDSServerAddress() string GetGlobaStatusAPI() string GenerateDpToken(mesh, appname string) (string, error) - GenerateZoneIngressToken(name string) (string, error) + GenerateZoneIngressToken(zone string) (string, error) } diff --git a/test/framework/k8s_controlplane.go b/test/framework/k8s_controlplane.go index 442b7511ad60..dc2fecd75233 100644 --- a/test/framework/k8s_controlplane.go +++ b/test/framework/k8s_controlplane.go @@ -269,12 +269,12 @@ func (c *K8sControlPlane) GenerateDpToken(mesh, service string) (string, error) ) } -func (c *K8sControlPlane) GenerateZoneIngressToken(name string) (string, error) { +func (c *K8sControlPlane) GenerateZoneIngressToken(zone string) (string, error) { return http_helper.HTTPDoWithRetryE( c.t, "POST", fmt.Sprintf("http://localhost:%d/tokens/zone-ingress", c.portFwd.localAPIPort), - []byte(fmt.Sprintf(`{"name": "%s"}`, name)), + []byte(fmt.Sprintf(`{"zone": "%s"}`, zone)), map[string]string{"content-type": "application/json"}, 200, DefaultRetries, diff --git a/test/framework/universal_app.go b/test/framework/universal_app.go index a0de374a3e25..680badc127f0 100644 --- a/test/framework/universal_app.go +++ b/test/framework/universal_app.go @@ -46,10 +46,11 @@ networking: ZoneIngress = ` type: ZoneIngress name: ingress -address: {{ address }} -advertisedAddress: %s -advertisedPort: %d -port: %d +networking: + address: {{ address }} + advertisedAddress: %s + advertisedPort: %d + port: %d ` EchoServerDataplane = ` @@ -394,7 +395,7 @@ func (s *UniversalApp) CreateDP(token, cpAddress, appname, ip, dpyaml string, bu args = append(args, "--dns-enabled") } if ingress { - args = append(args, "--ingress") + args = append(args, "--proxy-type=ingress") } s.dpApp = NewSshApp(s.verbose, s.ports[sshPort], []string{}, args) } diff --git a/test/framework/universal_controlplane.go b/test/framework/universal_controlplane.go index cd232161d9ef..c1f840fb1b9d 100644 --- a/test/framework/universal_controlplane.go +++ b/test/framework/universal_controlplane.go @@ -74,12 +74,12 @@ func (c *UniversalControlPlane) GenerateDpToken(mesh, service string) (string, e }) } -func (c *UniversalControlPlane) GenerateZoneIngressToken(name string) (string, error) { +func (c *UniversalControlPlane) GenerateZoneIngressToken(zone string) (string, error) { return retry.DoWithRetryE(c.t, "generating DP token", DefaultRetries, DefaultTimeout, func() (string, error) { sshApp := NewSshApp(c.verbose, c.cluster.apps[AppModeCP].ports["22"], []string{}, []string{"curl", "--fail", "--show-error", "-H", "\"Content-Type: application/json\"", - "--data", fmt.Sprintf(`'{"name": "%s"}'`, name), + "--data", fmt.Sprintf(`'{"zone": "%s"}'`, zone), "http://localhost:5681/tokens/zone-ingress"}) if err := sshApp.Run(); err != nil { return "", err From 6965921223392ae9d2b24f94791fc33e915ac987 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Mon, 14 Jun 2021 03:41:03 +0700 Subject: [PATCH 15/19] chore(kuma-cp) fix tests Signed-off-by: Ilya Lobkov --- api/mesh/v1alpha1/dataplane_helpers.go | 2 +- .../cmd/completion/testdata/bash.golden | 106 +++++++++++- .../cmd/completion/testdata/zsh.golden | 45 ++++- app/kumactl/cmd/generate/generate.go | 1 + .../generate/generate_dataplane_token_test.go | 2 +- .../generate/generate_zoneingress_token.go | 44 +++++ .../generate_zoneingress_token_test.go | 98 +++++++++++ app/kumactl/cmd/get/get.go | 4 +- .../cmd/get/get_zone-ingresses_test.go | 163 ++++++++++++++++++ .../testdata/get-zone-ingresses.golden.json | 71 ++++++++ .../testdata/get-zone-ingresses.golden.txt | 3 + .../testdata/get-zone-ingresses.golden.yaml | 42 +++++ .../get-zone-ingresses.pagination.golden.txt | 4 + .../cmd/install/install_tracing_test.go | 2 + ...all-control-plane.with-ingress.golden.yaml | 19 +- app/kumactl/pkg/cmd/root_context.go | 10 ++ app/kumactl/pkg/tokens/client_test.go | 15 -- app/kumactl/pkg/tokens/zoneingress_client.go | 70 ++++++++ .../pkg/tokens/zoneingress_client_test.go | 90 ++++++++++ docs/cmd/kumactl/HELP.md | 4 +- pkg/config/app/kuma-dp/config.go | 5 +- pkg/config/app/kuma-dp/config_test.go | 5 +- .../testdata/default-config.golden.yaml | 1 + .../testdata/invalid-config.input.yaml | 1 + .../kuma-dp/testdata/valid-config.input.yaml | 1 + .../k8s/controllers/ingress_converter.go | 4 + .../testdata/ingress/01.dataplane.yaml | 9 +- .../testdata/ingress/02.dataplane.yaml | 9 +- .../testdata/ingress/03.dataplane.yaml | 5 +- .../testdata/ingress/04.dataplane.yaml | 9 +- .../testdata/ingress/05.dataplane.yaml | 9 +- .../testdata/ingress/06.dataplane.yaml | 9 +- 32 files changed, 803 insertions(+), 59 deletions(-) create mode 100644 app/kumactl/cmd/generate/generate_zoneingress_token.go create mode 100644 app/kumactl/cmd/generate/generate_zoneingress_token_test.go create mode 100644 app/kumactl/cmd/get/get_zone-ingresses_test.go create mode 100644 app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.json create mode 100644 app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.txt create mode 100644 app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.yaml create mode 100644 app/kumactl/cmd/get/testdata/get-zone-ingresses.pagination.golden.txt create mode 100644 app/kumactl/pkg/tokens/zoneingress_client.go create mode 100644 app/kumactl/pkg/tokens/zoneingress_client_test.go diff --git a/api/mesh/v1alpha1/dataplane_helpers.go b/api/mesh/v1alpha1/dataplane_helpers.go index c15b9e06f6f7..11e812022aeb 100644 --- a/api/mesh/v1alpha1/dataplane_helpers.go +++ b/api/mesh/v1alpha1/dataplane_helpers.go @@ -47,7 +47,7 @@ func (t ProxyType) IsValid() error { case DataplaneProxyType, IngressProxyType, GatewayProxyType: return nil } - return errors.New("Invalid proxy type") + return errors.Errorf("%s is not a valid proxy type", t) } func (d *Dataplane) ProxyType() ProxyType { diff --git a/app/kumactl/cmd/completion/testdata/bash.golden b/app/kumactl/cmd/completion/testdata/bash.golden index 3f6f746ec10d..b03ea9685bd2 100644 --- a/app/kumactl/cmd/completion/testdata/bash.golden +++ b/app/kumactl/cmd/completion/testdata/bash.golden @@ -759,12 +759,12 @@ _kumactl_generate_dataplane-token() flags+=("--name=") two_word_flags+=("--name") local_nonpersistent_flags+=("--name=") + flags+=("--proxy-type=") + two_word_flags+=("--proxy-type") + local_nonpersistent_flags+=("--proxy-type=") flags+=("--tag=") two_word_flags+=("--tag") local_nonpersistent_flags+=("--tag=") - flags+=("--type=") - two_word_flags+=("--type") - local_nonpersistent_flags+=("--type=") flags+=("--config-file=") two_word_flags+=("--config-file") flags+=("--log-level=") @@ -820,6 +820,37 @@ _kumactl_generate_tls-certificate() noun_aliases=() } +_kumactl_generate_zone-ingress-token() +{ + last_command="kumactl_generate_zone-ingress-token" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--zone=") + two_word_flags+=("--zone") + local_nonpersistent_flags+=("--zone=") + flags+=("--config-file=") + two_word_flags+=("--config-file") + flags+=("--log-level=") + two_word_flags+=("--log-level") + flags+=("--mesh=") + two_word_flags+=("--mesh") + two_word_flags+=("-m") + flags+=("--no-config") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + _kumactl_generate() { last_command="kumactl_generate" @@ -829,6 +860,7 @@ _kumactl_generate() commands=() commands+=("dataplane-token") commands+=("tls-certificate") + commands+=("zone-ingress-token") flags=() two_word_flags=() @@ -1863,6 +1895,72 @@ _kumactl_get_zone() noun_aliases=() } +_kumactl_get_zone-ingress() +{ + last_command="kumactl_get_zone-ingress" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--config-file=") + two_word_flags+=("--config-file") + flags+=("--log-level=") + two_word_flags+=("--log-level") + flags+=("--mesh=") + two_word_flags+=("--mesh") + two_word_flags+=("-m") + flags+=("--no-config") + flags+=("--output=") + two_word_flags+=("--output") + two_word_flags+=("-o") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_kumactl_get_zone-ingresses() +{ + last_command="kumactl_get_zone-ingresses" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--offset=") + two_word_flags+=("--offset") + flags+=("--size=") + two_word_flags+=("--size") + flags+=("--config-file=") + two_word_flags+=("--config-file") + flags+=("--log-level=") + two_word_flags+=("--log-level") + flags+=("--mesh=") + two_word_flags+=("--mesh") + two_word_flags+=("-m") + flags+=("--no-config") + flags+=("--output=") + two_word_flags+=("--output") + two_word_flags+=("-o") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + _kumactl_get_zones() { last_command="kumactl_get_zones" @@ -1936,6 +2034,8 @@ _kumactl_get() commands+=("traffic-trace") commands+=("traffic-traces") commands+=("zone") + commands+=("zone-ingress") + commands+=("zone-ingresses") commands+=("zones") flags=() diff --git a/app/kumactl/cmd/completion/testdata/zsh.golden b/app/kumactl/cmd/completion/testdata/zsh.golden index c7e1ba6da2ff..a86e0fcd9f8a 100644 --- a/app/kumactl/cmd/completion/testdata/zsh.golden +++ b/app/kumactl/cmd/completion/testdata/zsh.golden @@ -287,6 +287,7 @@ function _kumactl_generate { commands=( "dataplane-token:Generate Dataplane Token" "tls-certificate:Generate a TLS certificate" + "zone-ingress-token:Generate Zone Ingress Token" ) _describe "command" commands ;; @@ -299,14 +300,17 @@ function _kumactl_generate { tls-certificate) _kumactl_generate_tls-certificate ;; + zone-ingress-token) + _kumactl_generate_zone-ingress-token + ;; esac } function _kumactl_generate_dataplane-token { _arguments \ '--name[name of the Dataplane]:' \ + '--proxy-type[type of the Dataplane ("dataplane", "ingress")]:' \ '--tag[required tag values for dataplane (split values by comma to provide multiple values)]:' \ - '--type[type of the Dataplane ("dataplane", "ingress")]:' \ '--config-file[path to the configuration file to use]:' \ '--log-level[log level: one of off|info|debug]:' \ '(-m --mesh)'{-m,--mesh}'[mesh to use]:' \ @@ -325,6 +329,15 @@ function _kumactl_generate_tls-certificate { '--no-config[if set no config file and config directory will be created]' } +function _kumactl_generate_zone-ingress-token { + _arguments \ + '--zone[name of the zone where ingress resides]:' \ + '--config-file[path to the configuration file to use]:' \ + '--log-level[log level: one of off|info|debug]:' \ + '(-m --mesh)'{-m,--mesh}'[mesh to use]:' \ + '--no-config[if set no config file and config directory will be created]' +} + function _kumactl_get { local -a commands @@ -371,7 +384,9 @@ function _kumactl_get { "traffic-routes:Show TrafficRoute" "traffic-trace:Show a single TrafficTrace resource" "traffic-traces:Show TrafficTrace" - "zone:Show a single Retry resource" + "zone:Show a single Zone resource" + "zone-ingress:Show a single ZoneIngress resource" + "zone-ingresses:Show ZoneIngress" "zones:Show Zone" ) _describe "command" commands @@ -472,6 +487,12 @@ function _kumactl_get { zone) _kumactl_get_zone ;; + zone-ingress) + _kumactl_get_zone-ingress + ;; + zone-ingresses) + _kumactl_get_zone-ingresses + ;; zones) _kumactl_get_zones ;; @@ -783,6 +804,26 @@ function _kumactl_get_zone { '(-o --output)'{-o,--output}'[output format: one of table|yaml|json]:' } +function _kumactl_get_zone-ingress { + _arguments \ + '--config-file[path to the configuration file to use]:' \ + '--log-level[log level: one of off|info|debug]:' \ + '(-m --mesh)'{-m,--mesh}'[mesh to use]:' \ + '--no-config[if set no config file and config directory will be created]' \ + '(-o --output)'{-o,--output}'[output format: one of table|yaml|json]:' +} + +function _kumactl_get_zone-ingresses { + _arguments \ + '--offset[the offset that indicates starting element of the resources list to retrieve]:' \ + '--size[maximum number of elements to return]:' \ + '--config-file[path to the configuration file to use]:' \ + '--log-level[log level: one of off|info|debug]:' \ + '(-m --mesh)'{-m,--mesh}'[mesh to use]:' \ + '--no-config[if set no config file and config directory will be created]' \ + '(-o --output)'{-o,--output}'[output format: one of table|yaml|json]:' +} + function _kumactl_get_zones { _arguments \ '--offset[the offset that indicates starting element of the resources list to retrieve]:' \ diff --git a/app/kumactl/cmd/generate/generate.go b/app/kumactl/cmd/generate/generate.go index f9951efff2c7..f54a2e9b94f2 100644 --- a/app/kumactl/cmd/generate/generate.go +++ b/app/kumactl/cmd/generate/generate.go @@ -14,6 +14,7 @@ func NewGenerateCmd(pctx *kumactl_cmd.RootContext) *cobra.Command { } // sub-commands cmd.AddCommand(NewGenerateDataplaneTokenCmd(pctx)) + cmd.AddCommand(NewGenerateZoneIngressTokenCmd(pctx)) cmd.AddCommand(NewGenerateCertificateCmd(pctx)) return cmd } diff --git a/app/kumactl/cmd/generate/generate_dataplane_token_test.go b/app/kumactl/cmd/generate/generate_dataplane_token_test.go index 56313185ec9a..f688f913044e 100644 --- a/app/kumactl/cmd/generate/generate_dataplane_token_test.go +++ b/app/kumactl/cmd/generate/generate_dataplane_token_test.go @@ -76,7 +76,7 @@ var _ = Describe("kumactl generate dataplane-token", func() { result: "token-for-example-default--", }), Entry("for all arguments", testCase{ - args: []string{"generate", "dataplane-token", "--mesh=demo", "--name=example", "--type=dataplane", "--tag", "kuma.io/service=web"}, + args: []string{"generate", "dataplane-token", "--mesh=demo", "--name=example", "--proxy-type=dataplane", "--tag", "kuma.io/service=web"}, result: "token-for-example-demo-kuma.io/service=web-dataplane", }), ) diff --git a/app/kumactl/cmd/generate/generate_zoneingress_token.go b/app/kumactl/cmd/generate/generate_zoneingress_token.go new file mode 100644 index 000000000000..6b4e48d6bfdd --- /dev/null +++ b/app/kumactl/cmd/generate/generate_zoneingress_token.go @@ -0,0 +1,44 @@ +package generate + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + + kumactl_cmd "github.com/kumahq/kuma/app/kumactl/pkg/cmd" +) + +type generateZoneIngressTokenContext struct { + *kumactl_cmd.RootContext + + args struct { + zone string + } +} + +func NewGenerateZoneIngressTokenCmd(pctx *kumactl_cmd.RootContext) *cobra.Command { + ctx := &generateZoneIngressTokenContext{RootContext: pctx} + cmd := &cobra.Command{ + Use: "zone-ingress-token", + Short: "Generate Zone Ingress Token", + Long: `Generate Zone Ingress Token that is used to prove Zone Ingress identity.`, + Example: ` +Generate token bound by zone +$ kumactl generate zone-ingress-token --zone zone-1 +`, + RunE: func(cmd *cobra.Command, _ []string) error { + client, err := pctx.CurrentZoneIngressTokenClient() + if err != nil { + return errors.Wrap(err, "failed to create zone ingress token client") + } + + token, err := client.Generate(ctx.args.zone) + if err != nil { + return errors.Wrap(err, "failed to generate a zone ingress token") + } + _, err = cmd.OutOrStdout().Write([]byte(token)) + return err + }, + } + cmd.Flags().StringVar(&ctx.args.zone, "zone", "", "name of the zone where ingress resides") + return cmd +} diff --git a/app/kumactl/cmd/generate/generate_zoneingress_token_test.go b/app/kumactl/cmd/generate/generate_zoneingress_token_test.go new file mode 100644 index 000000000000..999eb53ee507 --- /dev/null +++ b/app/kumactl/cmd/generate/generate_zoneingress_token_test.go @@ -0,0 +1,98 @@ +package generate_test + +import ( + "bytes" + "errors" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + "github.com/spf13/cobra" + + kumactl_resources "github.com/kumahq/kuma/app/kumactl/pkg/resources" + + "github.com/kumahq/kuma/app/kumactl/cmd" + kumactl_cmd "github.com/kumahq/kuma/app/kumactl/pkg/cmd" + "github.com/kumahq/kuma/app/kumactl/pkg/tokens" + config_proto "github.com/kumahq/kuma/pkg/config/app/kumactl/v1alpha1" +) + +type staticZoneIngressTokenGenerator struct { + err error +} + +var _ tokens.ZoneIngressTokenClient = &staticZoneIngressTokenGenerator{} + +func (s *staticZoneIngressTokenGenerator) Generate(zone string) (string, error) { + if s.err != nil { + return "", s.err + } + return fmt.Sprintf("token-for-%s", zone), nil +} + +var _ = Describe("kumactl generate zone-ingress-token", func() { + + var rootCmd *cobra.Command + var buf *bytes.Buffer + var generator *staticZoneIngressTokenGenerator + var ctx *kumactl_cmd.RootContext + + BeforeEach(func() { + generator = &staticZoneIngressTokenGenerator{} + ctx = &kumactl_cmd.RootContext{ + Runtime: kumactl_cmd.RootRuntime{ + NewZoneIngressTokenClient: func(*config_proto.ControlPlaneCoordinates_ApiServer) (tokens.ZoneIngressTokenClient, error) { + return generator, nil + }, + NewAPIServerClient: kumactl_resources.NewAPIServerClient, + }, + } + + rootCmd = cmd.NewRootCmd(ctx) + buf = &bytes.Buffer{} + rootCmd.SetOut(buf) + }) + + type testCase struct { + args []string + result string + } + DescribeTable("should generate token", + func(given testCase) { + // when + rootCmd.SetArgs(given.args) + err := rootCmd.Execute() + + // then + Expect(err).ToNot(HaveOccurred()) + + // and + Expect(buf.String()).To(Equal(given.result)) + }, + Entry("for zone", testCase{ + args: []string{"generate", "zone-ingress-token", "--zone=my-zone"}, + result: "token-for-my-zone", + }), + Entry("for empty zone", testCase{ + args: []string{"generate", "zone-ingress-token"}, + result: "token-for-", + }), + ) + + It("should write error when generating token fails", func() { + // setup + generator.err = errors.New("could not connect to API") + + // when + rootCmd.SetArgs([]string{"generate", "zone-ingress-token", "--zone=example"}) + err := rootCmd.Execute() + + // then + Expect(err).To(HaveOccurred()) + + // and + Expect(buf.String()).To(Equal("Error: failed to generate a zone ingress token: could not connect to API\n")) + }) + +}) diff --git a/app/kumactl/cmd/get/get.go b/app/kumactl/cmd/get/get.go index 5f1459557c75..3c7827ccabf7 100644 --- a/app/kumactl/cmd/get/get.go +++ b/app/kumactl/cmd/get/get.go @@ -36,6 +36,7 @@ func NewGetCmd(pctx *kumactl_cmd.RootContext) *cobra.Command { cmd.AddCommand(NewGetResourcesCmd(pctx, "secrets", core_system.SecretType, BasicResourceTablePrinter)) cmd.AddCommand(NewGetResourcesCmd(pctx, "global-secrets", core_system.GlobalSecretType, BasicGlobalResourceTablePrinter)) cmd.AddCommand(WithPaginationArgs(NewGetResourcesCmd(pctx, "zones", core_system.ZoneType, printZones), &pctx.ListContext)) + cmd.AddCommand(WithPaginationArgs(NewGetResourcesCmd(pctx, "zone-ingresses", core_mesh.ZoneIngressType, BasicGlobalResourceTablePrinter), &pctx.ListContext)) cmd.AddCommand(NewGetResourceCmd(pctx, "mesh", core_mesh.MeshType, printMeshes)) cmd.AddCommand(NewGetResourceCmd(pctx, "dataplane", core_mesh.DataplaneType, printDataplanes)) @@ -52,7 +53,8 @@ func NewGetCmd(pctx *kumactl_cmd.RootContext) *cobra.Command { cmd.AddCommand(NewGetResourceCmd(pctx, "timeout", core_mesh.TimeoutType, BasicResourceTablePrinter)) cmd.AddCommand(NewGetResourceCmd(pctx, "secret", core_system.SecretType, BasicResourceTablePrinter)) cmd.AddCommand(NewGetResourceCmd(pctx, "global-secret", core_system.GlobalSecretType, BasicGlobalResourceTablePrinter)) - cmd.AddCommand(NewGetResourceCmd(pctx, "zone", core_mesh.RetryType, printZones)) + cmd.AddCommand(NewGetResourceCmd(pctx, "zone", core_system.ZoneType, printZones)) + cmd.AddCommand(NewGetResourceCmd(pctx, "zone-ingress", core_mesh.ZoneIngressType, BasicGlobalResourceTablePrinter)) return cmd } diff --git a/app/kumactl/cmd/get/get_zone-ingresses_test.go b/app/kumactl/cmd/get/get_zone-ingresses_test.go new file mode 100644 index 000000000000..b9c6ff7d5242 --- /dev/null +++ b/app/kumactl/cmd/get/get_zone-ingresses_test.go @@ -0,0 +1,163 @@ +package get_test + +import ( + "bytes" + "context" + "io/ioutil" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + gomega_types "github.com/onsi/gomega/types" + "github.com/spf13/cobra" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + kumactl_resources "github.com/kumahq/kuma/app/kumactl/pkg/resources" + core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + + "github.com/kumahq/kuma/app/kumactl/cmd" + kumactl_cmd "github.com/kumahq/kuma/app/kumactl/pkg/cmd" + config_proto "github.com/kumahq/kuma/pkg/config/app/kumactl/v1alpha1" + core_model "github.com/kumahq/kuma/pkg/core/resources/model" + core_store "github.com/kumahq/kuma/pkg/core/resources/store" + memory_resources "github.com/kumahq/kuma/pkg/plugins/resources/memory" + test_model "github.com/kumahq/kuma/pkg/test/resources/model" +) + +var _ = Describe("kumactl get zone-ingresses", func() { + + zoneIngresses := []*core_mesh.ZoneIngressResource{ + { + Spec: &mesh_proto.ZoneIngress{ + Networking: &mesh_proto.ZoneIngress_Networking{ + Address: "1.1.1.1", + Port: 10001, + AdvertisedAddress: "2.2.2.2", + AdvertisedPort: 20002, + }, + AvailableServices: []*mesh_proto.ZoneIngress_AvailableService{ + {Mesh: "mesh-1", Tags: map[string]string{mesh_proto.ServiceTag: "svc-1"}}, + {Mesh: "mesh-2", Tags: map[string]string{mesh_proto.ServiceTag: "svc-2"}}, + {Mesh: "mesh-3", Tags: map[string]string{mesh_proto.ServiceTag: "svc-3"}}, + }, + }, + Meta: &test_model.ResourceMeta{ + Name: "ingress-zone-1", + }, + }, + { + Spec: &mesh_proto.ZoneIngress{ + Zone: "us-east", + Networking: &mesh_proto.ZoneIngress_Networking{ + Address: "3.3.3.3", + Port: 30003, + AdvertisedAddress: "4.4.4.4", + AdvertisedPort: 40004, + }, + AvailableServices: []*mesh_proto.ZoneIngress_AvailableService{ + {Mesh: "mesh-3", Tags: map[string]string{mesh_proto.ServiceTag: "svc-2"}}, + {Mesh: "mesh-4", Tags: map[string]string{mesh_proto.ServiceTag: "svc-3"}}, + {Mesh: "mesh-5", Tags: map[string]string{mesh_proto.ServiceTag: "svc-4"}}, + }, + }, + Meta: &test_model.ResourceMeta{ + Name: "ingress-zone-2", + }, + }, + } + + Describe("GetZoneIngressCmd", func() { + + var rootCtx *kumactl_cmd.RootContext + var rootCmd *cobra.Command + var buf *bytes.Buffer + var store core_store.ResourceStore + rootTime, _ := time.Parse(time.RFC3339, "2008-04-27T16:05:36.995Z") + BeforeEach(func() { + // setup + rootCtx = &kumactl_cmd.RootContext{ + Runtime: kumactl_cmd.RootRuntime{ + Now: func() time.Time { return rootTime }, + NewResourceStore: func(*config_proto.ControlPlaneCoordinates_ApiServer) (core_store.ResourceStore, error) { + return store, nil + }, + NewAPIServerClient: kumactl_resources.NewAPIServerClient, + }, + } + + store = core_store.NewPaginationStore(memory_resources.NewStore()) + + for _, cb := range zoneIngresses { + err := store.Create(context.Background(), cb, core_store.CreateBy(core_model.MetaToResourceKey(cb.GetMeta()))) + Expect(err).ToNot(HaveOccurred()) + } + + rootCmd = cmd.NewRootCmd(rootCtx) + buf = &bytes.Buffer{} + rootCmd.SetOut(buf) + }) + + type testCase struct { + outputFormat string + goldenFile string + pagination string + matcher func(interface{}) gomega_types.GomegaMatcher + } + + DescribeTable("kumactl get zone-ingresses -o table|json|yaml", + func(given testCase) { + // given + rootCmd.SetArgs(append([]string{ + "--config-file", filepath.Join("..", "testdata", "sample-kumactl.config.yaml"), + "get", "zone-ingresses"}, given.outputFormat, given.pagination)) + + // when + err := rootCmd.Execute() + // then + Expect(err).ToNot(HaveOccurred()) + + // when + expected, err := ioutil.ReadFile(filepath.Join("testdata", given.goldenFile)) + // then + Expect(err).ToNot(HaveOccurred()) + // and + Expect(buf.String()).To(given.matcher(expected)) + }, + Entry("should support Table output by default", testCase{ + outputFormat: "", + goldenFile: "get-zone-ingresses.golden.txt", + matcher: func(expected interface{}) gomega_types.GomegaMatcher { + return WithTransform(strings.TrimSpace, Equal(strings.TrimSpace(string(expected.([]byte))))) + }, + }), + Entry("should support Table output explicitly", testCase{ + outputFormat: "-otable", + goldenFile: "get-zone-ingresses.golden.txt", + matcher: func(expected interface{}) gomega_types.GomegaMatcher { + return WithTransform(strings.TrimSpace, Equal(strings.TrimSpace(string(expected.([]byte))))) + }, + }), + Entry("should support pagination", testCase{ + outputFormat: "-otable", + goldenFile: "get-zone-ingresses.pagination.golden.txt", + pagination: "--size=1", + matcher: func(expected interface{}) gomega_types.GomegaMatcher { + return WithTransform(strings.TrimSpace, Equal(strings.TrimSpace(string(expected.([]byte))))) + }, + }), + Entry("should support JSON output", testCase{ + outputFormat: "-ojson", + goldenFile: "get-zone-ingresses.golden.json", + matcher: MatchJSON, + }), + Entry("should support YAML output", testCase{ + outputFormat: "-oyaml", + goldenFile: "get-zone-ingresses.golden.yaml", + matcher: MatchYAML, + }), + ) + }) +}) diff --git a/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.json b/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.json new file mode 100644 index 000000000000..61433b7447d3 --- /dev/null +++ b/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.json @@ -0,0 +1,71 @@ +{ + "total": 2, + "items": [ + { + "type": "ZoneIngress", + "name": "ingress-zone-1", + "creationTime": "0001-01-01T00:00:00Z", + "modificationTime": "0001-01-01T00:00:00Z", + "networking": { + "address": "1.1.1.1", + "advertisedAddress": "2.2.2.2", + "port": 10001, + "advertisedPort": 20002 + }, + "availableServices": [ + { + "tags": { + "kuma.io/service": "svc-1" + }, + "mesh": "mesh-1" + }, + { + "tags": { + "kuma.io/service": "svc-2" + }, + "mesh": "mesh-2" + }, + { + "tags": { + "kuma.io/service": "svc-3" + }, + "mesh": "mesh-3" + } + ] + }, + { + "type": "ZoneIngress", + "name": "ingress-zone-2", + "creationTime": "0001-01-01T00:00:00Z", + "modificationTime": "0001-01-01T00:00:00Z", + "zone": "us-east", + "networking": { + "address": "3.3.3.3", + "advertisedAddress": "4.4.4.4", + "port": 30003, + "advertisedPort": 40004 + }, + "availableServices": [ + { + "tags": { + "kuma.io/service": "svc-2" + }, + "mesh": "mesh-3" + }, + { + "tags": { + "kuma.io/service": "svc-3" + }, + "mesh": "mesh-4" + }, + { + "tags": { + "kuma.io/service": "svc-4" + }, + "mesh": "mesh-5" + } + ] + } + ], + "next": null +} diff --git a/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.txt b/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.txt new file mode 100644 index 000000000000..15226d7e0375 --- /dev/null +++ b/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.txt @@ -0,0 +1,3 @@ +NAME AGE +ingress-zone-1 292y +ingress-zone-2 292y diff --git a/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.yaml b/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.yaml new file mode 100644 index 000000000000..adb4a530420e --- /dev/null +++ b/app/kumactl/cmd/get/testdata/get-zone-ingresses.golden.yaml @@ -0,0 +1,42 @@ +items: + - availableServices: + - mesh: mesh-1 + tags: + kuma.io/service: svc-1 + - mesh: mesh-2 + tags: + kuma.io/service: svc-2 + - mesh: mesh-3 + tags: + kuma.io/service: svc-3 + creationTime: "0001-01-01T00:00:00Z" + modificationTime: "0001-01-01T00:00:00Z" + name: ingress-zone-1 + networking: + address: 1.1.1.1 + advertisedAddress: 2.2.2.2 + advertisedPort: 20002 + port: 10001 + type: ZoneIngress + - availableServices: + - mesh: mesh-3 + tags: + kuma.io/service: svc-2 + - mesh: mesh-4 + tags: + kuma.io/service: svc-3 + - mesh: mesh-5 + tags: + kuma.io/service: svc-4 + creationTime: "0001-01-01T00:00:00Z" + modificationTime: "0001-01-01T00:00:00Z" + name: ingress-zone-2 + networking: + address: 3.3.3.3 + advertisedAddress: 4.4.4.4 + advertisedPort: 40004 + port: 30003 + type: ZoneIngress + zone: us-east +next: null +total: 2 diff --git a/app/kumactl/cmd/get/testdata/get-zone-ingresses.pagination.golden.txt b/app/kumactl/cmd/get/testdata/get-zone-ingresses.pagination.golden.txt new file mode 100644 index 000000000000..48508f9a64f5 --- /dev/null +++ b/app/kumactl/cmd/get/testdata/get-zone-ingresses.pagination.golden.txt @@ -0,0 +1,4 @@ +NAME AGE +ingress-zone-1 292y + +Rerun command with --offset=1 argument to retrieve more resources diff --git a/app/kumactl/cmd/install/install_tracing_test.go b/app/kumactl/cmd/install/install_tracing_test.go index cdf5304d0f68..81de436a9cb1 100644 --- a/app/kumactl/cmd/install/install_tracing_test.go +++ b/app/kumactl/cmd/install/install_tracing_test.go @@ -2,6 +2,7 @@ package install_test import ( "bytes" + "fmt" "path/filepath" . "github.com/onsi/ginkgo" @@ -39,6 +40,7 @@ var _ = Describe("kumactl install tracing", func() { // then Expect(err).ToNot(HaveOccurred()) + fmt.Println(stderr.String()) Expect(stderr.Bytes()).To(BeNil()) // and output matches golden files diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml index a3ad8f5452c3..40aca174b89f 100644 --- a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml +++ b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml @@ -779,7 +779,7 @@ spec: spec: serviceAccountName: kuma-control-plane nodeSelector: - + kubernetes.io/arch: amd64 kubernetes.io/os: linux containers: @@ -881,7 +881,6 @@ spec: metadata: annotations: kuma.io/ingress: enabled - kuma.io/mesh: default labels: app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma @@ -889,7 +888,7 @@ spec: spec: serviceAccountName: kuma-ingress nodeSelector: - + kubernetes.io/arch: amd64 kubernetes.io/os: linux containers: @@ -968,7 +967,7 @@ metadata: name: kuma-admission-mutating-webhook-configuration namespace: kuma-system labels: - + app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma webhooks: @@ -1017,8 +1016,8 @@ webhooks: - trafficpermissions - trafficroutes - traffictraces - - + + sideEffects: None - name: kuma-injector.kuma.io failurePolicy: Ignore @@ -1045,7 +1044,7 @@ metadata: name: kuma-validating-webhook-configuration namespace: kuma-system labels: - + app.kubernetes.io/name: kuma app.kubernetes.io/instance: kuma webhooks: @@ -1080,8 +1079,8 @@ webhooks: - trafficroutes - traffictraces - zones - - + + sideEffects: None - name: service.validator.kuma-admission.kuma.io failurePolicy: Ignore @@ -1124,4 +1123,4 @@ webhooks: - DELETE resources: - secrets - sideEffects: None + sideEffects: None \ No newline at end of file diff --git a/app/kumactl/pkg/cmd/root_context.go b/app/kumactl/pkg/cmd/root_context.go index 9254225035e4..161b8e7b811d 100644 --- a/app/kumactl/pkg/cmd/root_context.go +++ b/app/kumactl/pkg/cmd/root_context.go @@ -32,6 +32,7 @@ type RootRuntime struct { NewZoneOverviewClient func(*config_proto.ControlPlaneCoordinates_ApiServer) (kumactl_resources.ZoneOverviewClient, error) NewServiceOverviewClient func(*config_proto.ControlPlaneCoordinates_ApiServer) (kumactl_resources.ServiceOverviewClient, error) NewDataplaneTokenClient func(*config_proto.ControlPlaneCoordinates_ApiServer) (tokens.DataplaneTokenClient, error) + NewZoneIngressTokenClient func(*config_proto.ControlPlaneCoordinates_ApiServer) (tokens.ZoneIngressTokenClient, error) NewAPIServerClient func(*config_proto.ControlPlaneCoordinates_ApiServer) (kumactl_resources.ApiServerClient, error) } @@ -66,6 +67,7 @@ func DefaultRootContext() *RootContext { NewZoneOverviewClient: kumactl_resources.NewZoneOverviewClient, NewServiceOverviewClient: kumactl_resources.NewServiceOverviewClient, NewDataplaneTokenClient: tokens.NewDataplaneTokenClient, + NewZoneIngressTokenClient: tokens.NewZoneIngressTokenClient, NewAPIServerClient: kumactl_resources.NewAPIServerClient, }, TypeArgs: map[string]core_model.ResourceType{ @@ -197,6 +199,14 @@ func (rc *RootContext) CurrentDataplaneTokenClient() (tokens.DataplaneTokenClien return rc.Runtime.NewDataplaneTokenClient(controlPlane.Coordinates.ApiServer) } +func (rc *RootContext) CurrentZoneIngressTokenClient() (tokens.ZoneIngressTokenClient, error) { + controlPlane, err := rc.CurrentControlPlane() + if err != nil { + return nil, err + } + return rc.Runtime.NewZoneIngressTokenClient(controlPlane.Coordinates.ApiServer) +} + func (rc *RootContext) IsFirstTimeUsage() bool { if rc.Args.ConfigFile != "" { return !util_files.FileExists(rc.Args.ConfigFile) diff --git a/app/kumactl/pkg/tokens/client_test.go b/app/kumactl/pkg/tokens/client_test.go index a52304313fe7..c205f44a1c95 100644 --- a/app/kumactl/pkg/tokens/client_test.go +++ b/app/kumactl/pkg/tokens/client_test.go @@ -10,8 +10,6 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" - "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" - "github.com/kumahq/kuma/app/kumactl/pkg/tokens" config_kumactl "github.com/kumahq/kuma/pkg/config/app/kumactl/v1alpha1" "github.com/kumahq/kuma/pkg/tokens/builtin/issuer" @@ -31,19 +29,6 @@ func (s *staticTokenIssuer) Validate(token issuer.Token, meshName string) (issue return issuer.DataplaneIdentity{}, errors.New("not implemented") } -type zoneIngressStaticTokenIssuer struct { -} - -var _ zoneingress.TokenIssuer = &zoneIngressStaticTokenIssuer{} - -func (z *zoneIngressStaticTokenIssuer) Generate(identity zoneingress.Identity) (zoneingress.Token, error) { - return fmt.Sprintf("token-for-%s", identity.Zone), nil -} - -func (z *zoneIngressStaticTokenIssuer) Validate(token zoneingress.Token) (zoneingress.Identity, error) { - return zoneingress.Identity{}, errors.New("not implemented") -} - var _ = Describe("Tokens Client", func() { var server *httptest.Server diff --git a/app/kumactl/pkg/tokens/zoneingress_client.go b/app/kumactl/pkg/tokens/zoneingress_client.go new file mode 100644 index 000000000000..36344609e516 --- /dev/null +++ b/app/kumactl/pkg/tokens/zoneingress_client.go @@ -0,0 +1,70 @@ +package tokens + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" + + kumactl_client "github.com/kumahq/kuma/app/kumactl/pkg/client" + kumactl_config "github.com/kumahq/kuma/pkg/config/app/kumactl/v1alpha1" + error_types "github.com/kumahq/kuma/pkg/core/rest/errors/types" + "github.com/kumahq/kuma/pkg/tokens/builtin/server/types" + util_http "github.com/kumahq/kuma/pkg/util/http" +) + +func NewZoneIngressTokenClient(config *kumactl_config.ControlPlaneCoordinates_ApiServer) (ZoneIngressTokenClient, error) { + client, err := kumactl_client.ApiServerClient(config) + if err != nil { + return nil, err + } + return &httpZoneIngressTokenClient{ + client: client, + }, nil +} + +type ZoneIngressTokenClient interface { + Generate(zone string) (string, error) +} + +type httpZoneIngressTokenClient struct { + client util_http.Client +} + +var _ ZoneIngressTokenClient = &httpZoneIngressTokenClient{} + +func (h *httpZoneIngressTokenClient) Generate(zone string) (string, error) { + tokenReq := &types.ZoneIngressTokenRequest{ + Zone: zone, + } + reqBytes, err := json.Marshal(tokenReq) + if err != nil { + return "", errors.Wrap(err, "could not marshal token request to json") + } + req, err := http.NewRequest("POST", "/tokens/zone-ingress", bytes.NewReader(reqBytes)) + if err != nil { + return "", errors.Wrap(err, "could not construct the request") + } + req.Header.Set("content-type", "application/json") + resp, err := h.client.Do(req) + if err != nil { + return "", errors.Wrap(err, "could not execute the request") + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "could not read a body of the request") + } + if resp.StatusCode != 200 { + kumaErr := error_types.Error{} + if err := json.Unmarshal(body, &kumaErr); err == nil { + if kumaErr.Title != "" && kumaErr.Details != "" { + return "", &kumaErr + } + } + return "", errors.Errorf("(%d): %s", resp.StatusCode, body) + } + return string(body), nil +} diff --git a/app/kumactl/pkg/tokens/zoneingress_client_test.go b/app/kumactl/pkg/tokens/zoneingress_client_test.go new file mode 100644 index 000000000000..f9fec91bccb7 --- /dev/null +++ b/app/kumactl/pkg/tokens/zoneingress_client_test.go @@ -0,0 +1,90 @@ +package tokens_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + + "github.com/emicklei/go-restful" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + "github.com/kumahq/kuma/pkg/tokens/builtin/zoneingress" + + "github.com/kumahq/kuma/app/kumactl/pkg/tokens" + config_kumactl "github.com/kumahq/kuma/pkg/config/app/kumactl/v1alpha1" + tokens_server "github.com/kumahq/kuma/pkg/tokens/builtin/server" +) + +type zoneIngressStaticTokenIssuer struct { +} + +var _ zoneingress.TokenIssuer = &zoneIngressStaticTokenIssuer{} + +func (z *zoneIngressStaticTokenIssuer) Generate(identity zoneingress.Identity) (zoneingress.Token, error) { + return fmt.Sprintf("token-for-%s", identity.Zone), nil +} + +func (z *zoneIngressStaticTokenIssuer) Validate(token zoneingress.Token) (zoneingress.Identity, error) { + return zoneingress.Identity{}, errors.New("not implemented") +} + +var _ = Describe("Zone Ingress Tokens Client", func() { + + var server *httptest.Server + + BeforeEach(func() { + container := restful.NewContainer() + container.Add(tokens_server.NewWebservice(&staticTokenIssuer{}, &zoneIngressStaticTokenIssuer{})) + server = httptest.NewServer(container.ServeMux) + }) + + AfterEach(func() { + server.Close() + }) + + It("should return a token", func() { + // given + client, err := tokens.NewZoneIngressTokenClient(&config_kumactl.ControlPlaneCoordinates_ApiServer{ + Url: server.URL, + }) + Expect(err).ToNot(HaveOccurred()) + + // wait for server + Eventually(func() error { + _, err := client.Generate("my-zone-1") + return err + }, "5s", "100ms").ShouldNot(HaveOccurred()) + + // when + token, err := client.Generate("my-zone-1") + + // then + Expect(err).ToNot(HaveOccurred()) + Expect(token).To(Equal("token-for-my-zone-1")) + }) + + It("should return an error when status code is different than 200", func() { + // given + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + mux.HandleFunc("/tokens/zone-ingress", func(writer http.ResponseWriter, req *http.Request) { + defer GinkgoRecover() + writer.WriteHeader(500) + _, err := writer.Write([]byte("Internal Server Error")) + Expect(err).ToNot(HaveOccurred()) + }) + client, err := tokens.NewZoneIngressTokenClient(&config_kumactl.ControlPlaneCoordinates_ApiServer{ + Url: server.URL, + }) + Expect(err).ToNot(HaveOccurred()) + + // when + _, err = client.Generate("my-zone-2") + + // then + Expect(err).To(MatchError("(500): Internal Server Error")) + }) +}) diff --git a/docs/cmd/kumactl/HELP.md b/docs/cmd/kumactl/HELP.md index 77f5ba5dbf5c..d2c343a1cb00 100644 --- a/docs/cmd/kumactl/HELP.md +++ b/docs/cmd/kumactl/HELP.md @@ -449,7 +449,9 @@ Available Commands: traffic-routes Show TrafficRoute traffic-trace Show a single TrafficTrace resource traffic-traces Show TrafficTrace - zone Show a single Retry resource + zone Show a single Zone resource + zone-ingress Show a single ZoneIngress resource + zone-ingresses Show ZoneIngress zones Show Zone Flags: diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index 13a52aafc2a4..5806f0676870 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -27,6 +27,7 @@ func DefaultConfig() Config { Name: "", // Dataplane name must be set explicitly AdminPort: config_types.MustPortRange(30001, config_types.MaxPort), // by default, automatically choose a free port for Envoy Admin interface DrainTime: 30 * time.Second, + ProxyType: "dataplane", }, DataplaneRuntime: DataplaneRuntime{ BinaryPath: "envoy", @@ -112,7 +113,7 @@ type Dataplane struct { // Dataplane name. Name string `yaml:"name,omitempty" envconfig:"kuma_dataplane_name"` // ProxyType defines mode which should be used, supported values: 'dataplane', 'ingress' - ProxyType string `yaml:"isIngress,omitempty" envconfig:"kuma_dataplane_proxy_type"` + ProxyType string `yaml:"proxyType,omitempty" envconfig:"kuma_dataplane_proxy_type"` // Port (or range of ports to choose from) for Envoy Admin API to listen on. // Empty value indicates that Envoy Admin API should not be exposed over TCP. // Format: "9901 | 9901-9999 | 9901- | -9901". @@ -188,7 +189,7 @@ func (d *Dataplane) Sanitize() { func (d *Dataplane) Validate() (errs error) { proxyType := mesh_proto.ProxyType(d.ProxyType) if err := proxyType.IsValid(); err != nil { - errs = multierr.Append(errs, err) + errs = multierr.Append(errs, errors.Wrap(err, ".ProxyType is not valid")) } if d.Mesh == "" && proxyType != mesh_proto.IngressProxyType { errs = multierr.Append(errs, errors.Errorf(".Mesh must be non-empty")) diff --git a/pkg/config/app/kuma-dp/config_test.go b/pkg/config/app/kuma-dp/config_test.go index fbe6970e2052..506e3f76c4cf 100644 --- a/pkg/config/app/kuma-dp/config_test.go +++ b/pkg/config/app/kuma-dp/config_test.go @@ -1,6 +1,7 @@ package kumadp_test import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -60,6 +61,7 @@ var _ = Describe("Config", func() { "KUMA_DATAPLANE_NAME": "example", "KUMA_DATAPLANE_ADMIN_PORT": "2345", "KUMA_DATAPLANE_DRAIN_TIME": "60s", + "KUMA_DATAPLANE_PROXY_TYPE": "ingress", "KUMA_DATAPLANE_RUNTIME_BINARY_PATH": "envoy.sh", "KUMA_DATAPLANE_RUNTIME_CONFIG_DIR": "/var/run/envoy", "KUMA_DATAPLANE_RUNTIME_TOKEN_PATH": "/tmp/token", @@ -132,6 +134,7 @@ var _ = Describe("Config", func() { err := config.Load(filepath.Join("testdata", "invalid-config.input.yaml"), &cfg) // then - Expect(err.Error()).To(Equal(`Invalid configuration: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty`)) + fmt.Println(err.Error()) + Expect(err.Error()).To(Equal(`Invalid configuration: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty`)) }) }) diff --git a/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml b/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml index 6888e5001881..7bc3f0497086 100644 --- a/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml +++ b/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml @@ -8,6 +8,7 @@ controlPlane: dataplane: drainTime: 30s bootstrapVersion: "" + proxyType: dataplane dataplaneRuntime: binaryPath: envoy dns: diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml index 98ac9882a809..157920296b34 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml @@ -13,5 +13,6 @@ dataplane: # # adminPort: 82345 drainTime: 0 + proxyType: not-a-proxy dataplaneRuntime: binaryPath: diff --git a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml index 4dd9b5efe8ff..25468e9f6cc6 100644 --- a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml @@ -8,6 +8,7 @@ dataplane: name: example adminPort: 2345 drainTime: 60s + proxyType: ingress dataplaneRuntime: binaryPath: envoy.sh configDir: /var/run/envoy diff --git a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go index e30b0607672d..26c668e390c0 100644 --- a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go @@ -34,6 +34,10 @@ func (p *PodConverter) IngressFor(zoneIngress *mesh_proto.ZoneIngress, pod *kube return errors.Errorf("generated %d inbound interfaces, expected 1. Interfaces: %v", len(ifaces), ifaces) } + if zoneIngress.Networking == nil { + zoneIngress.Networking = &mesh_proto.ZoneIngress_Networking{} + } + zoneIngress.Networking.Address = pod.Status.PodIP zoneIngress.Networking.Port = ifaces[0].Port diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml index ae24421c747b..d858a920d036 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml @@ -1,7 +1,8 @@ metadata: creationTimestamp: null spec: - address: 192.168.0.1 - advertisedAddress: kuma-ingress.com - advertisedPort: 10001 - port: 10001 + networking: + address: 192.168.0.1 + advertisedAddress: kuma-ingress.com + advertisedPort: 10001 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml index 2145f8335e80..86fa5fc44aa8 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml @@ -1,7 +1,8 @@ metadata: creationTimestamp: null spec: - address: 192.168.0.1 - advertisedAddress: 192.168.100.1 - advertisedPort: 10001 - port: 10001 + networking: + address: 192.168.0.1 + advertisedAddress: 192.168.100.1 + advertisedPort: 10001 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml index eb249041edb5..1b076fd466c8 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml @@ -1,5 +1,6 @@ metadata: creationTimestamp: null spec: - address: 192.168.0.1 - port: 10001 + networking: + address: 192.168.0.1 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml index 73509704d540..87e89877c91e 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml @@ -1,7 +1,8 @@ metadata: creationTimestamp: null spec: - address: 192.168.0.1 - advertisedAddress: 34.72.129.131 - advertisedPort: 12345 - port: 10001 + networking: + address: 192.168.0.1 + advertisedAddress: 34.72.129.131 + advertisedPort: 12345 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml index e6f1c123af54..75c533fe8446 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml @@ -1,7 +1,8 @@ metadata: creationTimestamp: null spec: - address: 192.168.0.1 - advertisedAddress: 10.128.15.193 - advertisedPort: 12345 - port: 10001 + networking: + address: 192.168.0.1 + advertisedAddress: 10.128.15.193 + advertisedPort: 12345 + port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml index a2988744b5fe..429d8c83ce6a 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml @@ -1,7 +1,8 @@ metadata: creationTimestamp: null spec: - address: 192.168.0.1 - advertisedAddress: custom-address.com - advertisedPort: 1234 - port: 10001 + networking: + address: 192.168.0.1 + advertisedAddress: custom-address.com + advertisedPort: 1234 + port: 10001 From 316358fbb160bde5963449093dd06f859aa248de Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Mon, 14 Jun 2021 05:35:52 +0700 Subject: [PATCH 16/19] chore(kuma-cp) zone ingress insights Signed-off-by: Ilya Lobkov --- api/mesh/v1alpha1/zone_ingress.pb.go | 4 +- api/mesh/v1alpha1/zoneingress_overview.pb.go | 179 ++++++++++++++++++ api/mesh/v1alpha1/zoneingress_overview.proto | 16 ++ pkg/api-server/server.go | 6 + .../zoneingress_overview_endpoints.go | 107 +++++++++++ .../resources/apis/mesh/zoneingres_insight.go | 93 +++++++++ .../apis/mesh/zoneingress_overview.go | 120 ++++++++++++ .../server/callbacks/dataplane_status_sink.go | 7 + .../callbacks/dataplane_status_tracker.go | 6 +- 9 files changed, 531 insertions(+), 7 deletions(-) create mode 100644 api/mesh/v1alpha1/zoneingress_overview.pb.go create mode 100644 api/mesh/v1alpha1/zoneingress_overview.proto create mode 100644 pkg/api-server/zoneingress_overview_endpoints.go create mode 100644 pkg/core/resources/apis/mesh/zoneingres_insight.go create mode 100644 pkg/core/resources/apis/mesh/zoneingress_overview.go diff --git a/api/mesh/v1alpha1/zone_ingress.pb.go b/api/mesh/v1alpha1/zone_ingress.pb.go index 7e8e0e72dc29..4717d44f68cd 100644 --- a/api/mesh/v1alpha1/zone_ingress.pb.go +++ b/api/mesh/v1alpha1/zone_ingress.pb.go @@ -107,8 +107,8 @@ type ZoneIngress_Networking struct { // Address on which inbound listener will be exposed Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` - // AdvertisedAddress defines IP or DNS name on which ZoneIngress is accessible - // to other Kuma clusters. + // AdvertisedAddress defines IP or DNS name on which ZoneIngress is + // accessible to other Kuma clusters. AdvertisedAddress string `protobuf:"bytes,2,opt,name=advertisedAddress,proto3" json:"advertisedAddress,omitempty"` // Port of the inbound interface that will forward requests to the service. Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` diff --git a/api/mesh/v1alpha1/zoneingress_overview.pb.go b/api/mesh/v1alpha1/zoneingress_overview.pb.go new file mode 100644 index 000000000000..bb22038e0de9 --- /dev/null +++ b/api/mesh/v1alpha1/zoneingress_overview.pb.go @@ -0,0 +1,179 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.23.0 +// protoc v3.14.0 +// source: mesh/v1alpha1/zoneingress_overview.proto + +package v1alpha1 + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// ZoneIngressOverview defines the projected state of a ZoneIngress. +type ZoneIngressOverview struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ZoneIngress *ZoneIngress `protobuf:"bytes,1,opt,name=zone_ingress,json=zoneIngress,proto3" json:"zone_ingress,omitempty"` + ZoneIngressInsight *DataplaneInsight `protobuf:"bytes,2,opt,name=zone_ingress_insight,json=zoneIngressInsight,proto3" json:"zone_ingress_insight,omitempty"` +} + +func (x *ZoneIngressOverview) Reset() { + *x = ZoneIngressOverview{} + if protoimpl.UnsafeEnabled { + mi := &file_mesh_v1alpha1_zoneingress_overview_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ZoneIngressOverview) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ZoneIngressOverview) ProtoMessage() {} + +func (x *ZoneIngressOverview) ProtoReflect() protoreflect.Message { + mi := &file_mesh_v1alpha1_zoneingress_overview_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ZoneIngressOverview.ProtoReflect.Descriptor instead. +func (*ZoneIngressOverview) Descriptor() ([]byte, []int) { + return file_mesh_v1alpha1_zoneingress_overview_proto_rawDescGZIP(), []int{0} +} + +func (x *ZoneIngressOverview) GetZoneIngress() *ZoneIngress { + if x != nil { + return x.ZoneIngress + } + return nil +} + +func (x *ZoneIngressOverview) GetZoneIngressInsight() *DataplaneInsight { + if x != nil { + return x.ZoneIngressInsight + } + return nil +} + +var File_mesh_v1alpha1_zoneingress_overview_proto protoreflect.FileDescriptor + +var file_mesh_v1alpha1_zoneingress_overview_proto_rawDesc = []byte{ + 0x0a, 0x28, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, + 0x7a, 0x6f, 0x6e, 0x65, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x6f, 0x76, 0x65, 0x72, + 0x76, 0x69, 0x65, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x6b, 0x75, 0x6d, 0x61, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x1a, 0x20, + 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x7a, 0x6f, + 0x6e, 0x65, 0x5f, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x25, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, + 0x64, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x5f, 0x69, 0x6e, 0x73, 0x69, 0x67, 0x68, + 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb1, 0x01, 0x0a, 0x13, 0x5a, 0x6f, 0x6e, 0x65, + 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x4f, 0x76, 0x65, 0x72, 0x76, 0x69, 0x65, 0x77, 0x12, + 0x42, 0x0a, 0x0c, 0x7a, 0x6f, 0x6e, 0x65, 0x5f, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, + 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, + 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x0b, 0x7a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x12, 0x56, 0x0a, 0x14, 0x7a, 0x6f, 0x6e, 0x65, 0x5f, 0x69, 0x6e, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x5f, 0x69, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x24, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, + 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x52, 0x12, 0x7a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x42, 0x2a, 0x5a, 0x28, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, + 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_mesh_v1alpha1_zoneingress_overview_proto_rawDescOnce sync.Once + file_mesh_v1alpha1_zoneingress_overview_proto_rawDescData = file_mesh_v1alpha1_zoneingress_overview_proto_rawDesc +) + +func file_mesh_v1alpha1_zoneingress_overview_proto_rawDescGZIP() []byte { + file_mesh_v1alpha1_zoneingress_overview_proto_rawDescOnce.Do(func() { + file_mesh_v1alpha1_zoneingress_overview_proto_rawDescData = protoimpl.X.CompressGZIP(file_mesh_v1alpha1_zoneingress_overview_proto_rawDescData) + }) + return file_mesh_v1alpha1_zoneingress_overview_proto_rawDescData +} + +var file_mesh_v1alpha1_zoneingress_overview_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_mesh_v1alpha1_zoneingress_overview_proto_goTypes = []interface{}{ + (*ZoneIngressOverview)(nil), // 0: kuma.mesh.v1alpha1.ZoneIngressOverview + (*ZoneIngress)(nil), // 1: kuma.mesh.v1alpha1.ZoneIngress + (*DataplaneInsight)(nil), // 2: kuma.mesh.v1alpha1.DataplaneInsight +} +var file_mesh_v1alpha1_zoneingress_overview_proto_depIdxs = []int32{ + 1, // 0: kuma.mesh.v1alpha1.ZoneIngressOverview.zone_ingress:type_name -> kuma.mesh.v1alpha1.ZoneIngress + 2, // 1: kuma.mesh.v1alpha1.ZoneIngressOverview.zone_ingress_insight:type_name -> kuma.mesh.v1alpha1.DataplaneInsight + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_mesh_v1alpha1_zoneingress_overview_proto_init() } +func file_mesh_v1alpha1_zoneingress_overview_proto_init() { + if File_mesh_v1alpha1_zoneingress_overview_proto != nil { + return + } + file_mesh_v1alpha1_zone_ingress_proto_init() + file_mesh_v1alpha1_dataplane_insight_proto_init() + if !protoimpl.UnsafeEnabled { + file_mesh_v1alpha1_zoneingress_overview_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ZoneIngressOverview); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_mesh_v1alpha1_zoneingress_overview_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_mesh_v1alpha1_zoneingress_overview_proto_goTypes, + DependencyIndexes: file_mesh_v1alpha1_zoneingress_overview_proto_depIdxs, + MessageInfos: file_mesh_v1alpha1_zoneingress_overview_proto_msgTypes, + }.Build() + File_mesh_v1alpha1_zoneingress_overview_proto = out.File + file_mesh_v1alpha1_zoneingress_overview_proto_rawDesc = nil + file_mesh_v1alpha1_zoneingress_overview_proto_goTypes = nil + file_mesh_v1alpha1_zoneingress_overview_proto_depIdxs = nil +} diff --git a/api/mesh/v1alpha1/zoneingress_overview.proto b/api/mesh/v1alpha1/zoneingress_overview.proto new file mode 100644 index 000000000000..f25b4cea4e7a --- /dev/null +++ b/api/mesh/v1alpha1/zoneingress_overview.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package kuma.mesh.v1alpha1; + +option go_package = "github.com/kumahq/kuma/api/mesh/v1alpha1"; + +import "mesh/v1alpha1/zone_ingress.proto"; +import "mesh/v1alpha1/dataplane_insight.proto"; + +// ZoneIngressOverview defines the projected state of a ZoneIngress. +message ZoneIngressOverview { + + ZoneIngress zone_ingress = 1; + + DataplaneInsight zone_ingress_insight = 2; +} diff --git a/pkg/api-server/server.go b/pkg/api-server/server.go index e7c5d1a7245e..39c0adbb1650 100644 --- a/pkg/api-server/server.go +++ b/pkg/api-server/server.go @@ -160,6 +160,12 @@ func addResourcesEndpoints(ws *restful.WebService, defs []definitions.ResourceWs zoneOverviewEndpoints.addFindEndpoint(ws) zoneOverviewEndpoints.addListEndpoint(ws) + zoneIngressOverviewEndpoints := zoneIngressOverviewEndpoints{ + resManager: resManager, + } + zoneIngressOverviewEndpoints.addFindEndpoint(ws) + zoneIngressOverviewEndpoints.addListEndpoint(ws) + serviceInsightEndpoints := serviceInsightEndpoints{ resourceEndpoints: resourceEndpoints{ mode: cfg.Mode, diff --git a/pkg/api-server/zoneingress_overview_endpoints.go b/pkg/api-server/zoneingress_overview_endpoints.go new file mode 100644 index 000000000000..8a1e6318ce6a --- /dev/null +++ b/pkg/api-server/zoneingress_overview_endpoints.go @@ -0,0 +1,107 @@ +package api_server + +import ( + "context" + + "github.com/emicklei/go-restful" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" + + "github.com/kumahq/kuma/pkg/core/resources/manager" + "github.com/kumahq/kuma/pkg/core/resources/model" + "github.com/kumahq/kuma/pkg/core/resources/model/rest" + "github.com/kumahq/kuma/pkg/core/resources/store" + rest_errors "github.com/kumahq/kuma/pkg/core/rest/errors" +) + +type zoneIngressOverviewEndpoints struct { + resManager manager.ResourceManager +} + +func (r *zoneIngressOverviewEndpoints) addFindEndpoint(ws *restful.WebService) { + ws.Route(ws.GET("/zoneingresses+insights/{name}").To(r.inspectZoneIngress). + Doc("Inspect a zone ingress"). + Param(ws.PathParameter("name", "Name of a zone ingress").DataType("string")). + Returns(200, "OK", nil). + Returns(404, "Not found", nil)) +} + +func (r *zoneIngressOverviewEndpoints) addListEndpoint(ws *restful.WebService) { + ws.Route(ws.GET("/zoneingresses+insights").To(r.inspectZoneIngresses). + Doc("Inspect all zone ingresses"). + Returns(200, "OK", nil)) +} + +func (r *zoneIngressOverviewEndpoints) inspectZoneIngress(request *restful.Request, response *restful.Response) { + name := request.PathParameter("name") + + overview, err := r.fetchOverview(request.Request.Context(), name) + if err != nil { + rest_errors.HandleError(response, err, "Could not retrieve a zone ingress overview") + return + } + + res := rest.From.Resource(overview) + if err := response.WriteAsJson(res); err != nil { + rest_errors.HandleError(response, err, "Could not retrieve a zone ingress overview") + } +} + +func (r *zoneIngressOverviewEndpoints) fetchOverview(ctx context.Context, name string) (*mesh.ZoneIngressOverviewResource, error) { + zoneIngress := mesh.NewZoneIngressResource() + if err := r.resManager.Get(ctx, zoneIngress, store.GetByKey(name, model.NoMesh)); err != nil { + return nil, err + } + + insight := mesh.NewZoneIngressInsightResource() + err := r.resManager.Get(ctx, insight, store.GetByKey(name, model.NoMesh)) + if err != nil && !store.IsResourceNotFound(err) { // It's fine to have zone ingress without insight + return nil, err + } + + return &mesh.ZoneIngressOverviewResource{ + Meta: zoneIngress.Meta, + Spec: &mesh_proto.ZoneIngressOverview{ + ZoneIngress: zoneIngress.Spec, + ZoneIngressInsight: insight.Spec, + }, + }, nil +} + +func (r *zoneIngressOverviewEndpoints) inspectZoneIngresses(request *restful.Request, response *restful.Response) { + page, err := pagination(request) + if err != nil { + rest_errors.HandleError(response, err, "Could not retrieve dataplane overviews") + return + } + + overviews, err := r.fetchOverviews(request.Request.Context(), page) + if err != nil { + rest_errors.HandleError(response, err, "Could not retrieve dataplane overviews") + return + } + + // pagination is not supported yet so we need to override pagination total items after retaining dataplanes + overviews.GetPagination().SetTotal(uint32(len(overviews.Items))) + restList := rest.From.ResourceList(&overviews) + restList.Next = nextLink(request, overviews.GetPagination().NextOffset) + if err := response.WriteAsJson(restList); err != nil { + rest_errors.HandleError(response, err, "Could not list dataplane overviews") + } +} + +func (r *zoneIngressOverviewEndpoints) fetchOverviews(ctx context.Context, p page) (mesh.ZoneIngressOverviewResourceList, error) { + zoneIngresses := mesh.ZoneIngressResourceList{} + if err := r.resManager.List(ctx, &zoneIngresses, store.ListByPage(p.size, p.offset)); err != nil { + return mesh.ZoneIngressOverviewResourceList{}, err + } + + // we cannot paginate insights since there is no guarantee that the elements will be the same as zone ingresses + insights := mesh.ZoneIngressInsightResourceList{} + if err := r.resManager.List(ctx, &insights); err != nil { + return mesh.ZoneIngressOverviewResourceList{}, err + } + + return mesh.NewZoneIngressOverviews(zoneIngresses, insights), nil +} diff --git a/pkg/core/resources/apis/mesh/zoneingres_insight.go b/pkg/core/resources/apis/mesh/zoneingres_insight.go new file mode 100644 index 000000000000..4a3c7987338b --- /dev/null +++ b/pkg/core/resources/apis/mesh/zoneingres_insight.go @@ -0,0 +1,93 @@ +package mesh + +import ( + "errors" + + "github.com/kumahq/kuma/pkg/core/resources/registry" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core/resources/model" +) + +const ( + ZoneIngressInsightType model.ResourceType = "ZoneIngressInsight" +) + +var _ model.Resource = &ZoneIngressInsightResource{} + +type ZoneIngressInsightResource struct { + Meta model.ResourceMeta + Spec *mesh_proto.DataplaneInsight +} + +func NewZoneIngressInsightResource() *ZoneIngressInsightResource { + return &ZoneIngressInsightResource{ + Spec: &mesh_proto.DataplaneInsight{}, + } +} + +func (t *ZoneIngressInsightResource) GetType() model.ResourceType { + return ZoneIngressInsightType +} +func (t *ZoneIngressInsightResource) GetMeta() model.ResourceMeta { + return t.Meta +} +func (t *ZoneIngressInsightResource) SetMeta(m model.ResourceMeta) { + t.Meta = m +} +func (t *ZoneIngressInsightResource) GetSpec() model.ResourceSpec { + return t.Spec +} +func (t *ZoneIngressInsightResource) SetSpec(spec model.ResourceSpec) error { + status, ok := spec.(*mesh_proto.DataplaneInsight) + if !ok { + return errors.New("invalid type of spec") + } else { + t.Spec = status + return nil + } +} +func (t *ZoneIngressInsightResource) Validate() error { + return nil +} +func (t *ZoneIngressInsightResource) Scope() model.ResourceScope { + return model.ScopeGlobal +} + +var _ model.ResourceList = &ZoneIngressInsightResourceList{} + +type ZoneIngressInsightResourceList struct { + Items []*ZoneIngressInsightResource + Pagination model.Pagination +} + +func (l *ZoneIngressInsightResourceList) GetItems() []model.Resource { + res := make([]model.Resource, len(l.Items)) + for i, elem := range l.Items { + res[i] = elem + } + return res +} +func (l *ZoneIngressInsightResourceList) GetItemType() model.ResourceType { + return ZoneIngressInsightType +} +func (l *ZoneIngressInsightResourceList) NewItem() model.Resource { + return NewZoneIngressInsightResource() +} +func (l *ZoneIngressInsightResourceList) AddItem(r model.Resource) error { + if trr, ok := r.(*ZoneIngressInsightResource); ok { + l.Items = append(l.Items, trr) + return nil + } else { + return model.ErrorInvalidItemType((*ZoneIngressInsightResource)(nil), r) + } +} + +func (l *ZoneIngressInsightResourceList) GetPagination() *model.Pagination { + return &l.Pagination +} + +func init() { + registry.RegisterType(NewZoneIngressInsightResource()) + registry.RegistryListType(&ZoneIngressInsightResourceList{}) +} diff --git a/pkg/core/resources/apis/mesh/zoneingress_overview.go b/pkg/core/resources/apis/mesh/zoneingress_overview.go new file mode 100644 index 000000000000..70f1277541be --- /dev/null +++ b/pkg/core/resources/apis/mesh/zoneingress_overview.go @@ -0,0 +1,120 @@ +package mesh + +import ( + "errors" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core/resources/model" +) + +const ( + ZoneIngressOverviewType model.ResourceType = "ZoneIngressOverview" +) + +var _ model.Resource = &ZoneIngressOverviewResource{} + +type ZoneIngressOverviewResource struct { + Meta model.ResourceMeta + Spec *mesh_proto.ZoneIngressOverview +} + +func NewZoneIngressOverviewResource() *ZoneIngressOverviewResource { + return &ZoneIngressOverviewResource{ + Spec: &mesh_proto.ZoneIngressOverview{}, + } +} + +func (t *ZoneIngressOverviewResource) GetType() model.ResourceType { + return ZoneIngressOverviewType +} + +func (t *ZoneIngressOverviewResource) GetMeta() model.ResourceMeta { + return t.Meta +} + +func (t *ZoneIngressOverviewResource) SetMeta(m model.ResourceMeta) { + t.Meta = m +} + +func (t *ZoneIngressOverviewResource) GetSpec() model.ResourceSpec { + return t.Spec +} + +func (t *ZoneIngressOverviewResource) SetSpec(spec model.ResourceSpec) error { + zoneIngressOverview, ok := spec.(*mesh_proto.ZoneIngressOverview) + if !ok { + return errors.New("invalid type of spec") + } else { + t.Spec = zoneIngressOverview + return nil + } +} + +func (t *ZoneIngressOverviewResource) Validate() error { + return nil +} + +func (t *ZoneIngressOverviewResource) Scope() model.ResourceScope { + return model.ScopeGlobal +} + +var _ model.ResourceList = &ZoneIngressOverviewResourceList{} + +type ZoneIngressOverviewResourceList struct { + Items []*ZoneIngressOverviewResource + Pagination model.Pagination +} + +func (l *ZoneIngressOverviewResourceList) GetItems() []model.Resource { + res := make([]model.Resource, len(l.Items)) + for i, elem := range l.Items { + res[i] = elem + } + return res +} + +func (l *ZoneIngressOverviewResourceList) GetItemType() model.ResourceType { + return ZoneIngressOverviewType +} +func (l *ZoneIngressOverviewResourceList) NewItem() model.Resource { + return NewZoneIngressOverviewResource() +} +func (l *ZoneIngressOverviewResourceList) AddItem(r model.Resource) error { + if trr, ok := r.(*ZoneIngressOverviewResource); ok { + l.Items = append(l.Items, trr) + return nil + } else { + return model.ErrorInvalidItemType((*ZoneIngressOverviewResource)(nil), r) + } +} + +func (l *ZoneIngressOverviewResourceList) GetPagination() *model.Pagination { + return &l.Pagination +} + +func NewZoneIngressOverviews(zoneIngresses ZoneIngressResourceList, insights ZoneIngressInsightResourceList) ZoneIngressOverviewResourceList { + insightsByKey := map[model.ResourceKey]*ZoneIngressInsightResource{} + for _, insight := range insights.Items { + insightsByKey[model.MetaToResourceKey(insight.Meta)] = insight + } + + var items []*ZoneIngressOverviewResource + for _, zoneIngress := range zoneIngresses.Items { + overview := ZoneIngressOverviewResource{ + Meta: zoneIngress.Meta, + Spec: &mesh_proto.ZoneIngressOverview{ + ZoneIngress: zoneIngress.Spec, + ZoneIngressInsight: nil, + }, + } + insight, exists := insightsByKey[model.MetaToResourceKey(overview.Meta)] + if exists { + overview.Spec.ZoneIngressInsight = insight.Spec + } + items = append(items, &overview) + } + return ZoneIngressOverviewResourceList{ + Pagination: zoneIngresses.Pagination, + Items: items, + } +} diff --git a/pkg/xds/server/callbacks/dataplane_status_sink.go b/pkg/xds/server/callbacks/dataplane_status_sink.go index 0d43d558d6e6..2d23340e1021 100644 --- a/pkg/xds/server/callbacks/dataplane_status_sink.go +++ b/pkg/xds/server/callbacks/dataplane_status_sink.go @@ -106,6 +106,13 @@ type dataplaneInsightStore struct { } func (s *dataplaneInsightStore) Upsert(dataplaneId core_model.ResourceKey, subscription *mesh_proto.DiscoverySubscription) error { + if dataplaneId.Mesh == core_model.NoMesh { + // hack in order to support ZoneIngress which is the only supported Global-scope ProxyType today + return manager.Upsert(s.resManager, dataplaneId, mesh_core.NewZoneIngressInsightResource(), func(resource core_model.Resource) { + insight := resource.(*mesh_core.ZoneIngressInsightResource) + insight.Spec.UpdateSubscription(subscription) + }) + } return manager.Upsert(s.resManager, dataplaneId, mesh_core.NewDataplaneInsightResource(), func(resource core_model.Resource) { insight := resource.(*mesh_core.DataplaneInsightResource) insight.Spec.UpdateSubscription(subscription) diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index ba3a7b202e92..53a6d22124f1 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -114,8 +114,6 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di state.mu.Lock() // write access to the per Dataplane info defer state.mu.Unlock() - md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) - // infer Dataplane id if state.dataplaneId == (core_model.ResourceKey{}) { if proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { @@ -124,9 +122,7 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di statusTrackerLog.Error(err, "failed to extract version out of the Envoy metadata", "streamid", streamID, "metadata", req.Metadata()) } // kick off async Dataplane status flusher - if md.GetProxyType() != mesh_proto.IngressProxyType { - go c.createStatusSink(state).Start(state.stop) - } + go c.createStatusSink(state).Start(state.stop) } else { statusTrackerLog.Error(err, "failed to parse Dataplane Id out of DiscoveryRequest", "streamid", streamID, "req", req) } From 6149378d661b800c83b26134303da83acfa7bdf6 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Mon, 14 Jun 2021 05:57:39 +0700 Subject: [PATCH 17/19] chore(kuma-cp) fix integration test Signed-off-by: Ilya Lobkov --- pkg/plugins/resources/k8s/consistent_kind_type_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/plugins/resources/k8s/consistent_kind_type_test.go b/pkg/plugins/resources/k8s/consistent_kind_type_test.go index 3b8081d62dd4..c0267509ee01 100644 --- a/pkg/plugins/resources/k8s/consistent_kind_type_test.go +++ b/pkg/plugins/resources/k8s/consistent_kind_type_test.go @@ -1,6 +1,7 @@ package k8s import ( + "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -12,9 +13,10 @@ import ( // Those types are not mapped directly to Kubernetes Resource var IgnoredTypes = map[model.ResourceType]bool{ - system.SecretType: true, - system.GlobalSecretType: true, - system.ConfigType: true, + system.SecretType: true, + system.GlobalSecretType: true, + system.ConfigType: true, + mesh.ZoneIngressInsightType: true, // uses DataplaneInsight under the hood } var _ = Describe("Consistent Kind Types", func() { From 9390514731560c774568d5426f0980be82b205e5 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Mon, 14 Jun 2021 15:09:17 +0700 Subject: [PATCH 18/19] chore(kuma-cp) review Signed-off-by: Ilya Lobkov --- pkg/core/resources/apis/mesh/zone_ingress_validator.go | 2 +- pkg/core/resources/apis/mesh/zone_ingress_validator_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator.go b/pkg/core/resources/apis/mesh/zone_ingress_validator.go index d8b5ae5e0e55..34ede80b956d 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_validator.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator.go @@ -7,7 +7,7 @@ import ( func (r *ZoneIngressResource) Validate() error { var err validators.ValidationError - err.Add(validateZoneIngress(validators.RootedAt("ingress"), r.Spec)) + err.Add(validateZoneIngress(validators.RootedAt("networking"), r.Spec)) return err.OrNil() } diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go index 13d0440ab186..f0ebb52b7a9c 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go @@ -119,7 +119,7 @@ var _ = Describe("Dataplane", func() { region: eu`, expected: ` violations: - - field: ingress.port + - field: networking.port message: port has to be defined`, }), Entry("invalid advertised address and port", testCase{ @@ -142,9 +142,9 @@ var _ = Describe("Dataplane", func() { region: eu`, expected: ` violations: - - field: ingress.advertisedAddress.address + - field: networking.advertisedAddress.address message: address has to be valid IP address or domain name - - field: ingress.advertisedPort + - field: networking.advertisedPort message: port has to be in range of [1, 65535]`, }), ) From 526a3db15ac6ffcc9cb84329a4a67cde3aa52991 Mon Sep 17 00:00:00 2001 From: Ilya Lobkov Date: Mon, 14 Jun 2021 17:18:25 +0700 Subject: [PATCH 19/19] chore(kuma-cp) review Signed-off-by: Ilya Lobkov --- .../apis/mesh/zone_ingress_validator.go | 39 ++++++++++--------- .../apis/mesh/zone_ingress_validator_test.go | 18 +++++++-- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator.go b/pkg/core/resources/apis/mesh/zone_ingress_validator.go index 34ede80b956d..b1814ffb81df 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_validator.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator.go @@ -7,42 +7,45 @@ import ( func (r *ZoneIngressResource) Validate() error { var err validators.ValidationError - err.Add(validateZoneIngress(validators.RootedAt("networking"), r.Spec)) + err.Add(r.validateNetworking(validators.RootedAt("networking"), r.Spec.GetNetworking())) + err.Add(r.validateAvailableServices(validators.RootedAt("availableService"), r.Spec.GetAvailableServices())) return err.OrNil() } -func validateZoneIngress(path validators.PathBuilder, ingress *mesh_proto.ZoneIngress) validators.ValidationError { - if ingress == nil { - return validators.ValidationError{} - } +func (r *ZoneIngressResource) validateNetworking(path validators.PathBuilder, networking *mesh_proto.ZoneIngress_Networking) validators.ValidationError { var err validators.ValidationError - if ingress.GetNetworking().GetAdvertisedAddress() == "" && ingress.GetNetworking().GetAdvertisedPort() != 0 { + if networking.GetAdvertisedAddress() == "" && networking.GetAdvertisedPort() != 0 { err.AddViolationAt(path.Field("advertisedAddress"), `has to be defined with advertisedPort`) } - if ingress.GetNetworking().GetAdvertisedPort() == 0 && ingress.GetNetworking().GetAdvertisedAddress() != "" { + if networking.GetAdvertisedPort() == 0 && networking.GetAdvertisedAddress() != "" { err.AddViolationAt(path.Field("advertisedPort"), `has to be defined with advertisedAddress`) } - if ingress.GetNetworking().GetAddress() != "" { - err.Add(validateAddress(path.Field("address"), ingress.GetNetworking().GetAddress())) + if networking.GetAddress() != "" { + err.Add(validateAddress(path.Field("address"), networking.GetAddress())) } - if ingress.GetNetworking().GetAdvertisedAddress() != "" { - err.Add(validateAddress(path.Field("advertisedAddress"), ingress.GetNetworking().GetAdvertisedAddress())) + if networking.GetAdvertisedAddress() != "" { + err.Add(validateAddress(path.Field("advertisedAddress"), networking.GetAdvertisedAddress())) } - if ingress.GetNetworking().GetPort() == 0 { + if networking.GetPort() == 0 { err.AddViolationAt(path.Field("port"), `port has to be defined`) } - if ingress.GetNetworking().GetPort() > 65535 { + if networking.GetPort() > 65535 { err.AddViolationAt(path.Field("port"), `port has to be in range of [1, 65535]`) } - if ingress.GetNetworking().GetAdvertisedPort() > 65535 { + if networking.GetAdvertisedPort() > 65535 { err.AddViolationAt(path.Field("advertisedPort"), `port has to be in range of [1, 65535]`) } - for i, ingressInterface := range ingress.GetAvailableServices() { - p := path.Field("availableService").Index(i) - if _, ok := ingressInterface.Tags[mesh_proto.ServiceTag]; !ok { + return err +} + +func (r *ZoneIngressResource) validateAvailableServices(path validators.PathBuilder, availableServices []*mesh_proto.ZoneIngress_AvailableService) validators.ValidationError { + var err validators.ValidationError + for i, availableService := range availableServices { + p := path.Index(i) + if _, ok := availableService.Tags[mesh_proto.ServiceTag]; !ok { err.AddViolationAt(p.Field("tags").Key(mesh_proto.ServiceTag), "cannot be empty") } - err.AddErrorAt(p.Field("tags"), validateTags(ingressInterface.GetTags())) + err.AddErrorAt(p.Field("tags"), validateTags(availableService.GetTags())) } return err } diff --git a/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go index f0ebb52b7a9c..0e56c7809794 100644 --- a/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go +++ b/pkg/core/resources/apis/mesh/zone_ingress_validator_test.go @@ -122,7 +122,7 @@ var _ = Describe("Dataplane", func() { - field: networking.port message: port has to be defined`, }), - Entry("invalid advertised address and port", testCase{ + Entry("invalid advertised address, port and available service", testCase{ dataplane: ` type: ZoneIngress name: zi-1 @@ -139,13 +139,25 @@ var _ = Describe("Dataplane", func() { - tags: kuma.io/service: web version: v2 - region: eu`, + region: eu + - tags: + version: v2 + region: eu + - tags: + kuma.io/service: "" + version: ""`, expected: ` violations: - field: networking.advertisedAddress.address message: address has to be valid IP address or domain name - field: networking.advertisedPort - message: port has to be in range of [1, 65535]`, + message: port has to be in range of [1, 65535] + - field: availableService[2].tags["kuma.io/service"] + message: cannot be empty + - field: availableService[3].tags.tags["kuma.io/service"] + message: tag value cannot be empty + - field: availableService[3].tags.tags["version"] + message: tag value cannot be empty`, }), )