Skip to content
This repository has been archived by the owner on Jul 11, 2023. It is now read-only.

Commit

Permalink
pkg/injector: Enable podIP proxying via meshconfig setting (#4701)
Browse files Browse the repository at this point in the history 
* pkg/injector: Enable podIP proxying via meshconfig setting

Fixes #4653. Allows customers to choose how their traffic gets proxied

Signed-off-by: Keith Mattix II <[email protected]>

* Fix linting and test errors

Signed-off-by: Keith Mattix II <[email protected]>

* Inject pod_ip into initcontainer

Signed-off-by: Keith Mattix II <[email protected]>

* Fix port in pod command

Signed-off-by: Keith Mattix II <[email protected]>

* Reduce leaky abstraction by not passing configurator to generateIPTables

Signed-off-by: Keith Mattix II <[email protected]>

* Use iptables -I to make the rules dynamic

Signed-off-by: Keith Mattix II <[email protected]>

* Fix array append

Signed-off-by: Keith Mattix II <[email protected]>
  • Loading branch information
@keithmattix
keithmattix committed May 3, 2022
1 parent 868c132 commit cbdcfe1
Show file tree
Hide file tree
Showing 9 changed files with 232 additions and 24 deletions.
3 changes: 2 additions & 1 deletion charts/osm/templates/preset-mesh-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ data:
"enablePrivilegedInitContainer": {{.Values.osm.enablePrivilegedInitContainer | mustToJson}},
"logLevel": {{.Values.osm.envoyLogLevel | mustToJson}},
"maxDataPlaneConnections": {{.Values.osm.maxDataPlaneConnections | mustToJson}},
"configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}}
"configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}},
"localProxyMode": {{.Values.osm.localProxyMode | mustToJson}}
},
"traffic": {
"enableEgress": {{.Values.osm.enableEgress | mustToJson}},
Expand Down
14 changes: 13 additions & 1 deletion pkg/injector/init_container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ import (
func getInitContainerSpec(containerName string, cfg configurator.Configurator, outboundIPRangeExclusionList []string,
outboundIPRangeInclusionList []string, outboundPortExclusionList []int,
inboundPortExclusionList []int, enablePrivilegedInitContainer bool, pullPolicy corev1.PullPolicy, networkInterfaceExclusionList []string) corev1.Container {
iptablesInitCommand := generateIptablesCommands(outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)
proxyMode := cfg.GetMeshConfig().Spec.Sidecar.LocalProxyMode
iptablesInitCommand := generateIptablesCommands(proxyMode, outboundIPRangeExclusionList, outboundIPRangeInclusionList, outboundPortExclusionList, inboundPortExclusionList, networkInterfaceExclusionList)

return corev1.Container{
Name: containerName,
Expand All @@ -32,5 +33,16 @@ func getInitContainerSpec(containerName string, cfg configurator.Configurator, o
"-c",
iptablesInitCommand,
},
Env: []corev1.EnvVar{
{
Name: "POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
APIVersion: "v1",
FieldPath: "status.podIP",
},
},
},
},
}
}
97 changes: 97 additions & 0 deletions pkg/injector/init_container_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"github.com/golang/mock/gomock"
corev1 "k8s.io/api/core/v1"

configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"

"github.com/openservicemesh/osm/pkg/configurator"
)

Expand All @@ -26,6 +28,89 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
Context("test getInitContainerSpec()", func() {
It("Creates init container without ip range exclusion list", func() {
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
Spec: configv1alpha2.MeshConfigSpec{
Sidecar: configv1alpha2.SidecarSpec{
LocalProxyMode: configv1alpha2.LocalProxyModeLocalhost,
},
},
}).Times(1)
privileged := privilegedFalse
actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)

expected := corev1.Container{
Name: "-container-name-",
Image: "-init-container-image-",
ImagePullPolicy: corev1.PullAlways,
Command: []string{"/bin/sh"},
Args: []string{
"-c",
`iptables-restore --noflush <<EOF
# OSM sidecar interception rules
*nat
:OSM_PROXY_INBOUND - [0:0]
:OSM_PROXY_IN_REDIRECT - [0:0]
:OSM_PROXY_OUTBOUND - [0:0]
:OSM_PROXY_OUT_REDIRECT - [0:0]
-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
COMMIT
EOF
`,
},
WorkingDir: "",
Resources: corev1.ResourceRequirements{},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
Privileged: &privilegedFalse,
RunAsNonRoot: &runAsNonRootFalse,
RunAsUser: &runAsUserID,
},
Env: []corev1.EnvVar{
{
Name: "POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
APIVersion: "v1",
FieldPath: "status.podIP",
},
},
},
},
Stdin: false,
StdinOnce: false,
TTY: false,
}

Expect(actual).To(Equal(expected))
})
It("Sets podIP DNAT rule if set in meshconfig", func() {
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
mockConfigurator.EXPECT().GetMeshConfig().Return(configv1alpha2.MeshConfig{
Spec: configv1alpha2.MeshConfigSpec{
Sidecar: configv1alpha2.SidecarSpec{
LocalProxyMode: configv1alpha2.LocalProxyModePodIP,
},
},
}).Times(1)
privileged := privilegedFalse
actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, nil, privileged, corev1.PullAlways, nil)

Expand Down Expand Up @@ -58,6 +143,7 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
COMMIT
EOF
Expand All @@ -75,6 +161,17 @@ EOF
RunAsNonRoot: &runAsNonRootFalse,
RunAsUser: &runAsUserID,
},
Env: []corev1.EnvVar{
{
Name: "POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
APIVersion: "v1",
FieldPath: "status.podIP",
},
},
},
},
Stdin: false,
StdinOnce: false,
TTY: false,
Expand Down
11 changes: 10 additions & 1 deletion pkg/injector/iptables.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"strconv"
"strings"

configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"

"github.com/openservicemesh/osm/pkg/constants"
)

Expand Down Expand Up @@ -60,7 +62,7 @@ var iptablesInboundStaticRules = []string{
}

// generateIptablesCommands generates a list of iptables commands to set up sidecar interception and redirection
func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
func generateIptablesCommands(proxyMode configv1alpha2.LocalProxyMode, outboundIPRangeExclusionList []string, outboundIPRangeInclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int, networkInterfaceExclusionList []string) string {
var rules strings.Builder

fmt.Fprintln(&rules, `# OSM sidecar interception rules
Expand Down Expand Up @@ -95,6 +97,13 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundIPR
// 3. Create outbound rules
cmds = append(cmds, iptablesOutboundStaticRules...)

if proxyMode == configv1alpha2.LocalProxyModePodIP {
// For envoy -> local service container proxying, send traffic to pod IP instead of localhost
// *Note: it is important to use the insert option '-I' instead of the append option '-A' to ensure the
// DNAT to the pod ip for envoy -> localhost traffic happens before the rule that redirects traffic to the proxy
cmds = append(cmds, fmt.Sprintf("-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner %d -j DNAT --to-destination $POD_IP", constants.EnvoyUID))
}

// Ignore outbound traffic in specified interfaces
for _, iface := range networkInterfaceExclusionList {
cmds = append(cmds, fmt.Sprintf("-A OSM_PROXY_OUTBOUND -o %s -j RETURN", iface))
Expand Down
37 changes: 36 additions & 1 deletion pkg/injector/iptables_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@ import (
"testing"

"github.com/stretchr/testify/assert"

configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
)

func TestGenerateIptablesCommands(t *testing.T) {
testCases := []struct {
name string
proxyMode configv1alpha2.LocalProxyMode
outboundIPRangeExclusions []string
outboundIPRangeInclusions []string
outboundPortExclusions []int
Expand Down Expand Up @@ -87,14 +90,46 @@ EOF
-A OSM_PROXY_OUTBOUND -j RETURN
COMMIT
EOF
`,
},
{
name: "proxy mode pod ip",
proxyMode: configv1alpha2.LocalProxyModePodIP,
expected: `iptables-restore --noflush <<EOF
# OSM sidecar interception rules
*nat
:OSM_PROXY_INBOUND - [0:0]
:OSM_PROXY_IN_REDIRECT - [0:0]
:OSM_PROXY_OUTBOUND - [0:0]
:OSM_PROXY_OUT_REDIRECT - [0:0]
-A OSM_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
-A PREROUTING -p tcp -j OSM_PROXY_INBOUND
-A OSM_PROXY_INBOUND -p tcp --dport 15010 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15901 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15902 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15903 -j RETURN
-A OSM_PROXY_INBOUND -p tcp --dport 15904 -j RETURN
-A OSM_PROXY_INBOUND -p tcp -j OSM_PROXY_IN_REDIRECT
-A OSM_PROXY_OUT_REDIRECT -p tcp -j REDIRECT --to-port 15001
-A OSM_PROXY_OUT_REDIRECT -p tcp --dport 15000 -j ACCEPT
-A OUTPUT -p tcp -j OSM_PROXY_OUTBOUND
-A OSM_PROXY_OUTBOUND -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1500 -j OSM_PROXY_IN_REDIRECT
-A OSM_PROXY_OUTBOUND -o lo -m owner ! --uid-owner 1500 -j RETURN
-A OSM_PROXY_OUTBOUND -m owner --uid-owner 1500 -j RETURN
-A OSM_PROXY_OUTBOUND -d 127.0.0.1/32 -j RETURN
-I OUTPUT -p tcp -o lo -d 127.0.0.1/32 -m owner --uid-owner 1500 -j DNAT --to-destination $POD_IP
-A OSM_PROXY_OUTBOUND -j OSM_PROXY_OUT_REDIRECT
COMMIT
EOF
`,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
a := assert.New(t)
actual := generateIptablesCommands(tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)

actual := generateIptablesCommands(tc.proxyMode, tc.outboundIPRangeExclusions, tc.outboundIPRangeInclusions, tc.outboundPortExclusions, tc.inboundPortExclusions, tc.networkInterfaceExclusions)
a.Equal(tc.expected, actual)
})
}
Expand Down
20 changes: 14 additions & 6 deletions tests/e2e/e2e_pod_client_server_test.go → ...2e/e2e_client_server_connectivity_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ import (

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"

"github.com/openservicemesh/osm/pkg/constants"
. "github.com/openservicemesh/osm/tests/framework"
)
Expand All @@ -25,26 +26,33 @@ var _ = OSMDescribe("Test HTTP traffic from 1 pod client -> 1 pod server",
func() {
Context("Test traffic flowing from client to server with a Kubernetes Service for the Source: HTTP", func() {
withSourceKubernetesService := true
testTraffic(withSourceKubernetesService)
testTraffic(withSourceKubernetesService, PodCommandDefault)
})

Context("Test traffic flowing from client to server without a Kubernetes Service for the Source: HTTP", func() {
// Prior iterations of OSM required that a source pod belong to a Kubernetes service
// for the Envoy proxy to be configured for outbound traffic to some remote server.
// This test ensures we test this scenario: client Pod is not associated w/ a service.
withSourceKubernetesService := false
testTraffic(withSourceKubernetesService)
testTraffic(withSourceKubernetesService, PodCommandDefault)
})

Context("Test traffic flowing from client to a server with a podIP bind", func() {
// Prior iterations of OSM didn't allow mesh services to bind to the podIP
// This test ensures that that behavior is configurable via MeshConfig
withSourceKubernetesService := true
testTraffic(withSourceKubernetesService, []string{"gunicorn", "-b", "$(POD_IP):80", "httpbin:app", "-k", "gevent"}, WithLocalProxyMode(v1alpha2.LocalProxyModePodIP))
})
})

func testTraffic(withSourceKubernetesService bool) {
func testTraffic(withSourceKubernetesService bool, destPodCommand []string, installOpts ...InstallOsmOpt) {
const sourceName = "client"
const destName = "server"
var ns = []string{sourceName, destName}

It("Tests HTTP traffic for client pod -> server pod", func() {
// Install OSM
Expect(Td.InstallOSM(Td.GetOSMInstallOpts())).To(Succeed())
Expect(Td.InstallOSM(Td.GetOSMInstallOpts(installOpts...))).To(Succeed())

// Create Test NS
for _, n := range ns {
Expand All @@ -53,7 +61,7 @@ func testTraffic(withSourceKubernetesService bool) {
}

// Get simple pod definitions for the HTTP server
svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName)
svcAccDef, podDef, svcDef, err := Td.GetOSSpecificHTTPBinPod(destName, destName, destPodCommand...)
Expect(err).NotTo(HaveOccurred())

_, err = Td.CreateServiceAccount(destName, &svcAccDef)
Expand Down
24 changes: 21 additions & 3 deletions tests/framework/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,13 +328,21 @@ nodeRegistration:
return nil
}

// WithLocalProxyMode sets the LocalProxyMode for OSM
func WithLocalProxyMode(mode configv1alpha2.LocalProxyMode) InstallOsmOpt {
return func(opts *InstallOSMOpts) {
opts.LocalProxyMode = mode
}
}

// GetOSMInstallOpts initializes install options for OSM
func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
func (td *OsmTestData) GetOSMInstallOpts(options ...InstallOsmOpt) InstallOSMOpts {
enablePrivilegedInitContainer := false
if td.DeployOnOpenShift {
enablePrivilegedInitContainer = true
}
return InstallOSMOpts{

baseOpts := InstallOSMOpts{
ControlPlaneNS: td.OsmNamespace,
CertManager: defaultCertManager,
ContainerRegistryLoc: td.CtrRegistryServer,
Expand Down Expand Up @@ -364,6 +372,12 @@ func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts {
EnablePrivilegedInitContainer: enablePrivilegedInitContainer,
EnableIngressBackendPolicy: true,
}

for _, opt := range options {
opt(&baseOpts)
}

return baseOpts
}

// LoadImagesToKind loads the list of images to the node for Kind clusters
Expand Down Expand Up @@ -415,7 +429,7 @@ func (td *OsmTestData) LoadImagesToKind(imageNames []string) error {
return nil
}

func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *configv1alpha2.MeshConfig) (defaultConfig *configv1alpha2.MeshConfig) {
func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *configv1alpha2.MeshConfig) *configv1alpha2.MeshConfig {
meshConfig.Spec.Traffic.EnableEgress = instOpts.EgressEnabled
meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode = instOpts.EnablePermissiveMode
meshConfig.Spec.Traffic.OutboundPortExclusionList = []int{}
Expand Down Expand Up @@ -491,6 +505,10 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error {
fmt.Sprintf("osm.enableReconciler=%v", instOpts.EnableReconciler),
)

if instOpts.LocalProxyMode != "" {
instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("osm.localProxyMode=%s", instOpts.LocalProxyMode))
}

switch instOpts.CertManager {
case "vault":
if err := td.installVault(instOpts); err != nil {
Expand Down
Loading

0 comments on commit cbdcfe1

Please sign in to comment.