Skip to content

Commit

Permalink
Merge pull request kubernetes-sigs#114 from njucjc/fix-nat
Browse files Browse the repository at this point in the history 
Fix: raven traffic ignore nat
  • Loading branch information
@njucjc
njucjc authored May 30, 2023
2 parents 004816d + 4ab1506 commit 4449a2c
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 21 deletions.
58 changes: 41 additions & 17 deletions pkg/networkengine/routedriver/vxlan/vxlan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ const (
)

var (
nonGatewayChainRuleSpec = []string{"-m", "set", "--match-set", ravenMarkSet, "dst", "-j", "MARK", "--set-mark", fmt.Sprintf("%d", ravenMark)}
gatewayChainRuleSpec = []string{"-m", "set", "--match-set", ravenMarkSet, "src", "-j", "MARK", "--set-mark", fmt.Sprintf("%d", ravenMark)}
nonGatewayChainRuleSpec = []string{"-m", "set", "--match-set", ravenMarkSet, "dst", "-m", "comment", "--comment", "raven traffic requiring certain mark", "-j", "MARK", "--set-mark", fmt.Sprintf("%d", ravenMark)}
gatewayChainRuleSpec = []string{"-m", "set", "--match-set", ravenMarkSet, "src", "-m", "comment", "--comment", "raven traffic requiring certain mark", "-j", "MARK", "--set-mark", fmt.Sprintf("%d", ravenMark)}
)

func init() {
Expand Down Expand Up @@ -102,7 +102,7 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
return fmt.Errorf("error create ip set: %s", err)
}

err = vx.ensureRavenMarkChain()
err = vx.ensureRavenChain()
if err != nil {
return fmt.Errorf("error ensure raven mark chain: %s", err)
}
Expand Down Expand Up @@ -138,23 +138,23 @@ func (vx *vxlan) Apply(network *types.Network, vpnDriverMTUFn func() (int, error
desiredRoutes = vx.calRouteOnGateway(network)
desiredFDBs = vx.calFDBOnGateway(network)

err = vx.deleteChainRuleOnNode(nonGatewayChainRuleSpec)
err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, nonGatewayChainRuleSpec)
if err != nil {
return fmt.Errorf("error deleting non gateway chain rule: %s", err)
}
err = vx.addChainRuleOnNode(gatewayChainRuleSpec)
err = vx.addChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, gatewayChainRuleSpec)
if err != nil {
return fmt.Errorf("error adding gateway chain rule: %s", err)
}
} else {
desiredRoutes = vx.calRouteOnNonGateway(network)
desiredFDBs = vx.calFDBOnNonGateway(network)

err = vx.deleteChainRuleOnNode(gatewayChainRuleSpec)
err = vx.deleteChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, gatewayChainRuleSpec)
if err != nil {
return fmt.Errorf("error deleting gateway chain rule: %s", err)
}
err = vx.addChainRuleOnNode(nonGatewayChainRuleSpec)
err = vx.addChainRuleOnNode(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, nonGatewayChainRuleSpec)
if err != nil {
return fmt.Errorf("error adding non gateway chain rule: %s", err)
}
Expand Down Expand Up @@ -234,16 +234,28 @@ func (vx *vxlan) Init() (err error) {
return
}

func (vx *vxlan) ensureRavenMarkChain() error {
func (vx *vxlan) ensureRavenChain() error {
// for raven mark
if err := vx.iptables.NewChainIfNotExist(iptablesutil.MangleTable, iptablesutil.RavenMarkChain); err != nil {
return fmt.Errorf("error create %s chain: %s", iptablesutil.RavenMarkChain, err)
}
if err := vx.iptables.AppendIfNotExists(iptablesutil.MangleTable, iptablesutil.PreRoutingChain, "-j", iptablesutil.RavenMarkChain); err != nil {
if err := vx.iptables.AppendIfNotExists(iptablesutil.MangleTable, iptablesutil.PreRoutingChain, "-m", "comment", "--comment", "raven traffic rules for mark", "-j", iptablesutil.RavenMarkChain); err != nil {
return fmt.Errorf("error adding chain %s rule: %s", iptablesutil.PreRoutingChain, err)
}
if err := vx.iptables.AppendIfNotExists(iptablesutil.MangleTable, iptablesutil.OutputChain, "-j", iptablesutil.RavenMarkChain); err != nil {
if err := vx.iptables.AppendIfNotExists(iptablesutil.MangleTable, iptablesutil.OutputChain, "-m", "comment", "--comment", "raven traffic rules for mark", "-j", iptablesutil.RavenMarkChain); err != nil {
return fmt.Errorf("error adding chain %s rule: %s", iptablesutil.OutputChain, err)
}
// for raven skip nat
if err := vx.iptables.NewChainIfNotExist(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain); err != nil {
return fmt.Errorf("error create %s chain: %s", iptablesutil.RavenPostRoutingChain, err)
}
if err := vx.iptables.InsertIfNotExists(iptablesutil.NatTable, iptablesutil.PostRoutingChain, 1, "-m", "comment", "--comment", "raven traffic should skip NAT", "-o", "raven0", "-j", iptablesutil.RavenPostRoutingChain); err != nil {
return fmt.Errorf("error adding chain %s rule: %s", iptablesutil.PostRoutingChain, err)
}
if err := vx.iptables.AppendIfNotExists(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain, "-j", "ACCEPT"); err != nil {
return fmt.Errorf("error adding chain %s rule: %s", iptablesutil.RavenPostRoutingChain, err)
}

return nil
}

Expand Down Expand Up @@ -458,7 +470,6 @@ func (vx *vxlan) Cleanup() error {
}

// Clean may be called more than one time, so we should ensure chain exists

err := vx.iptables.NewChainIfNotExist(iptablesutil.MangleTable, iptablesutil.RavenMarkChain)
if err != nil {
errList = errList.Append(fmt.Errorf("error ensure chain %s: %s", iptablesutil.RavenMarkChain, err))
Expand All @@ -476,6 +487,19 @@ func (vx *vxlan) Cleanup() error {
errList = errList.Append(fmt.Errorf("error deleting %s chain %s", iptablesutil.RavenMarkChain, err))
}

err = vx.iptables.NewChainIfNotExist(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain)
if err != nil {
errList = errList.Append(fmt.Errorf("error create %s chain: %s", iptablesutil.PostRoutingChain, err))
}
err = vx.iptables.DeleteIfExists(iptablesutil.NatTable, iptablesutil.PostRoutingChain, "-o", "raven0", "-j", iptablesutil.RavenPostRoutingChain)
if err != nil {
errList = errList.Append(fmt.Errorf("error deleting %s chain rule: %s", iptablesutil.PostRoutingChain, err))
}
err = vx.iptables.ClearAndDeleteChain(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain)
if err != nil {
errList = errList.Append(fmt.Errorf("error deleting %s chain %s", iptablesutil.RavenPostRoutingChain, err))
}

// Clean may be called more than one time, so we should ensure ip set exists
vx.ipset, err = ipsetutil.New(ravenMarkSet)
if err != nil {
Expand All @@ -492,16 +516,16 @@ func (vx *vxlan) Cleanup() error {
return errList.AsError()
}

func (vx *vxlan) deleteChainRuleOnNode(ruleSpec []string) error {
if err := vx.iptables.DeleteIfExists(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, ruleSpec...); err != nil {
return fmt.Errorf("error deleting chain %s rule %v: %s", iptablesutil.RavenMarkChain, ruleSpec, err)
func (vx *vxlan) deleteChainRuleOnNode(table string, chain string, ruleSpec []string) error {
if err := vx.iptables.DeleteIfExists(table, chain, ruleSpec...); err != nil {
return fmt.Errorf("error deleting table %s chain %s rule %v: %s", table, chain, ruleSpec, err)
}
return nil
}

func (vx *vxlan) addChainRuleOnNode(ruleSpec []string) error {
if err := vx.iptables.AppendIfNotExists(iptablesutil.MangleTable, iptablesutil.RavenMarkChain, ruleSpec...); err != nil {
return fmt.Errorf("error adding chain %s rule %v: %s", iptablesutil.RavenMarkChain, ruleSpec, err)
func (vx *vxlan) addChainRuleOnNode(table string, chain string, ruleSpec []string) error {
if err := vx.iptables.AppendIfNotExists(table, chain, ruleSpec...); err != nil {
return fmt.Errorf("error adding table %s chain %s rule %v: %s", table, chain, ruleSpec, err)
}
return nil
}
Expand Down
11 changes: 7 additions & 4 deletions pkg/networkengine/util/iptables/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,11 @@
package iptablesutil

const (
PreRoutingChain = "PREROUTING"
OutputChain = "OUTPUT"
RavenMarkChain = "RAVEN-MARK-CHAIN"
MangleTable = "mangle"
PreRoutingChain = "PREROUTING"
PostRoutingChain = "POSTROUTING"
OutputChain = "OUTPUT"
RavenMarkChain = "RAVEN-MARK-CHAIN"
RavenPostRoutingChain = "RAVEN-POSTROUTING"
MangleTable = "mangle"
NatTable = "nat"
)
16 changes: 16 additions & 0 deletions pkg/networkengine/util/iptables/iptables.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ type IPTablesInterface interface {
List(table, chain string) ([]string, error)
AppendIfNotExists(table, chain string, rulespec ...string) error
DeleteIfExists(table, chain string, rulespec ...string) error
InsertIfNotExists(table, chain string, pos int, rulespec ...string) error
}

type iptablesWrapper struct {
Expand Down Expand Up @@ -113,3 +114,18 @@ func (ipt *iptablesWrapper) DeleteIfExists(table, chain string, rulespec ...stri
}
return nil
}

func (ipt *iptablesWrapper) InsertIfNotExists(table, chain string, pos int, rulespec ...string) error {
exists, err := ipt.Exists(table, chain, rulespec...)
if err == nil && !exists {
err = ipt.Insert(table, chain, pos, rulespec...)
}
if err != nil {
klog.ErrorS(err, "error on iptables.Insert", "table", table, "chain", chain, "pos", pos, "rulespec", rulespec, "exists", exists)
return err
}
if klog.V(5).Enabled() {
klog.V(5).InfoS("iptables.Insert succeeded", "table", table, "chain", chain, "pos", pos, "rulespec", rulespec, "exists", exists)
}
return nil
}

0 comments on commit 4449a2c

Please sign in to comment.