Skip to content
This repository has been archived by the owner on Jul 11, 2023. It is now read-only.

Commit

Permalink
cli/verifier: verify basic HTTP route configs (#4682)
Browse files Browse the repository at this point in the history 
- Verifies HTTP inbound and outbound route
  configs and top level service domain

- Updates code to correctly translate k8s
  service to MeshService. Previously this
  would break when a named TargetPort is
  used for the service port.

- Exports reusable code in k8s pkg

Part of #4634

Signed-off-by: Shashank Ram <[email protected]>
  • Loading branch information
@shashankram
shashankram authored Apr 25, 2022
1 parent bc3ff99 commit 24a494b
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 12 deletions.
15 changes: 15 additions & 0 deletions pkg/cli/verifier/connectivity_pod_to_pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,21 @@ func TestRun(t *testing.T) {
ClusterIP: "10.96.15.1",
},
},
&corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "httpbin",
Namespace: "httpbin",
},
Subsets: []corev1.EndpointSubset{
{
Ports: []corev1.EndpointPort{
{
Port: 14001,
},
},
},
},
},
},
trafficAttr: TrafficAttribute{
SrcPod: &types.NamespacedName{Namespace: "curl", Name: "curl"},
Expand Down
145 changes: 135 additions & 10 deletions pkg/cli/verifier/envoy_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,20 @@ import (

mapset "github.com/deckarep/golang-set"
xds_listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
xds_route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"github.com/envoyproxy/go-control-plane/pkg/wellknown"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/utils/pointer"

configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"

"github.com/openservicemesh/osm/pkg/constants"
"github.com/openservicemesh/osm/pkg/envoy/lds"
"github.com/openservicemesh/osm/pkg/envoy/rds/route"
"github.com/openservicemesh/osm/pkg/k8s"
"github.com/openservicemesh/osm/pkg/service"
)

Expand Down Expand Up @@ -103,7 +107,7 @@ func (v *EnvoyConfigVerifier) Run() Result {

func (v *EnvoyConfigVerifier) verifySource() Result {
result := Result{
Context: fmt.Sprintf("Verify Envoy config on source for traffic: %s", v.configAttr),
Context: "Verify Envoy config on source",
}

config, err := v.configAttr.srcConfigGetter.Get()
Expand Down Expand Up @@ -141,7 +145,20 @@ func (v *EnvoyConfigVerifier) verifySource() Result {
return result
}

// Next, if the destination service is known, verify it has a matching filter chain
// Retrieve route configs
var routeConfigs []*xds_route.RouteConfiguration
if v.configAttr.trafficAttr.AppProtocol == constants.ProtocolHTTP {
configs := config.Routes.GetDynamicRouteConfigs()
for _, r := range configs {
routeConfig := &xds_route.RouteConfiguration{}
//nolint: errcheck
//#nosec G104: Errors unhandled
r.GetRouteConfig().UnmarshalTo(routeConfig)
routeConfigs = append(routeConfigs, routeConfig)
}
}

// Next, if the destination service is known, verify it has a matching filter chain and route
if v.configAttr.trafficAttr.DstService != nil {
dst := v.configAttr.trafficAttr.DstService
svc, err := v.kubeClient.CoreV1().Services(dst.Namespace).Get(context.Background(), dst.Name, metav1.GetOptions{})
Expand All @@ -155,6 +172,14 @@ func (v *EnvoyConfigVerifier) verifySource() Result {
result.Reason = fmt.Sprintf("Did not find matching outbound filter chain for service %q: %s", dst, err)
return result
}

if v.configAttr.trafficAttr.AppProtocol == constants.ProtocolHTTP {
if err := v.findHTTPRouteForService(svc, routeConfigs, true); err != nil {
result.Status = Failure
result.Reason = fmt.Sprintf("Did not find matching outbound route configuration for service %q: %s", dst, err)
return result
}
}
}

result.Status = Success
Expand Down Expand Up @@ -260,7 +285,7 @@ func getFilterForProtocol(protocol string) string {

func (v *EnvoyConfigVerifier) verifyDestination() Result {
result := Result{
Context: fmt.Sprintf("Verify Envoy config on destination for traffic: %s", v.configAttr),
Context: "Verify Envoy config on destination",
}

config, err := v.configAttr.dstConfigGetter.Get()
Expand Down Expand Up @@ -298,6 +323,19 @@ func (v *EnvoyConfigVerifier) verifyDestination() Result {
return result
}

// Retrieve route configs
var routeConfigs []*xds_route.RouteConfiguration
if v.configAttr.trafficAttr.AppProtocol == constants.ProtocolHTTP {
configs := config.Routes.GetDynamicRouteConfigs()
for _, r := range configs {
routeConfig := &xds_route.RouteConfiguration{}
//nolint: errcheck
//#nosec G104: Errors unhandled
r.GetRouteConfig().UnmarshalTo(routeConfig)
routeConfigs = append(routeConfigs, routeConfig)
}
}

// Next, if the destination service is known, verify it has a matching filter chain
if v.configAttr.trafficAttr.DstService != nil {
dst := v.configAttr.trafficAttr.DstService
Expand All @@ -312,24 +350,53 @@ func (v *EnvoyConfigVerifier) verifyDestination() Result {
result.Reason = fmt.Sprintf("Did not find matching inbound filter chain for service %q: %s", dst, err)
return result
}
if v.configAttr.trafficAttr.AppProtocol == constants.ProtocolHTTP {
if err := v.findHTTPRouteForService(svc, routeConfigs, false); err != nil {
result.Status = Failure
result.Reason = fmt.Sprintf("Did not find matching inbound route configuration for service %q: %s", dst, err)
return result
}
}
}

result.Status = Success
return result
}

func (v *EnvoyConfigVerifier) getDstMeshServicesForSvc(svc corev1.Service) ([]service.MeshService, error) {
endpoints, err := v.kubeClient.CoreV1().Endpoints(svc.Namespace).Get(context.Background(), svc.Name, metav1.GetOptions{})
if err != nil || endpoints == nil {
return nil, err
}

var meshServices []service.MeshService
for _, portSpec := range svc.Spec.Ports {
meshSvc := service.MeshService{
Namespace: svc.Namespace,
Name: svc.Name,
Port: uint16(portSpec.Port),
Protocol: pointer.StringDeref(portSpec.AppProtocol, constants.ProtocolHTTP),
}

// The endpoints for the kubernetes service carry information that allows
// us to retrieve the TargetPort for the MeshService.
meshSvc.TargetPort = k8s.GetTargetPortFromEndpoints(portSpec.Name, *endpoints)
meshServices = append(meshServices, meshSvc)
}
return meshServices, nil
}

func (v *EnvoyConfigVerifier) findInboundFilterChainForService(svc *corev1.Service, filterChains []*xds_listener.FilterChain) error {
if svc == nil {
return nil
}

for _, port := range svc.Spec.Ports {
meshSvc := service.MeshService{
Name: svc.Name,
Namespace: svc.Namespace,
Protocol: v.configAttr.trafficAttr.AppProtocol,
TargetPort: uint16(port.TargetPort.IntVal),
}
meshServices, err := v.getDstMeshServicesForSvc(*svc)
if len(meshServices) == 0 || err != nil {
return errors.Errorf("endpoints not found for service %s/%s, err: %s", svc.Namespace, svc.Name, err)
}

for _, meshSvc := range meshServices {
if err := findInboundFilterChainForServicePort(meshSvc, filterChains); err != nil {
return err
}
Expand Down Expand Up @@ -374,3 +441,61 @@ func findInboundFilterChainForServicePort(meshSvc service.MeshService, filterCha

return nil
}

func (v *EnvoyConfigVerifier) findHTTPRouteForService(svc *corev1.Service, routeConfigs []*xds_route.RouteConfiguration, isOutbound bool) error {
if svc == nil {
return nil
}

meshServices, err := v.getDstMeshServicesForSvc(*svc)
if len(meshServices) == 0 || err != nil {
return errors.Errorf("endpoints not found for service %s/%s, err: %s", svc.Namespace, svc.Name, err)
}

for _, meshSvc := range meshServices {
var desiredConfigName string
if isOutbound {
desiredConfigName = route.GetOutboundMeshRouteConfigNameForPort(int(meshSvc.Port))
} else {
desiredConfigName = route.GetInboundMeshRouteConfigNameForPort(int(meshSvc.TargetPort))
}

if err := findHTTPRouteConfig(routeConfigs, desiredConfigName, meshSvc.FQDN()); err != nil {
return err
}
}

return nil
}

func findHTTPRouteConfig(routeConfigs []*xds_route.RouteConfiguration, desireConfigName string, desiredDomain string) error {
var config *xds_route.RouteConfiguration

for _, c := range routeConfigs {
if c.Name == desireConfigName {
config = c
break
}
}

if config == nil {
return errors.Errorf("route configuration %s not found", desireConfigName)
}

// Look for the FQDN in the virtual hosts
var virtualHost *xds_route.VirtualHost
for _, vh := range config.VirtualHosts {
for _, domain := range vh.Domains {
if domain == desiredDomain {
virtualHost = vh
break
}
}
}

if virtualHost == nil {
return errors.Errorf("virtual host for domain %s not found", desiredDomain)
}

return nil
}
5 changes: 3 additions & 2 deletions pkg/k8s/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ func ServiceToMeshServices(c Controller, svc corev1.Service) []service.MeshServi
// us to retrieve the TargetPort for the MeshService.
endpoints, _ := c.GetEndpoints(meshSvc)
if endpoints != nil {
meshSvc.TargetPort = getTargetPortFromEndpoints(portSpec.Name, *endpoints)
meshSvc.TargetPort = GetTargetPortFromEndpoints(portSpec.Name, *endpoints)
} else {
log.Warn().Msgf("k8s service %s/%s does not have endpoints but is being represented as a MeshService", svc.Namespace, svc.Name)
}
Expand All @@ -368,7 +368,8 @@ func ServiceToMeshServices(c Controller, svc corev1.Service) []service.MeshServi
return meshServices
}

func getTargetPortFromEndpoints(endpointName string, endpoints corev1.Endpoints) (endpointPort uint16) {
// GetTargetPortFromEndpoints returns the endpoint port corresponding to the given endpoint name and endpoints
func GetTargetPortFromEndpoints(endpointName string, endpoints corev1.Endpoints) (endpointPort uint16) {
// Per https://pkg.go.dev/k8s.io/api/core/v1#ServicePort and
// https://pkg.go.dev/k8s.io/api/core/v1#EndpointPort, if a service has multiple
// ports, then ServicePort.Name must match EndpointPort.Name when considering
Expand Down

0 comments on commit 24a494b

Please sign in to comment.