From cf97e6032c38498d00ce95142e8ce52195277cdf Mon Sep 17 00:00:00 2001 From: M00nF1sh Date: Mon, 11 Apr 2022 10:58:32 -0700 Subject: [PATCH] record pod metadata and allocationTime in IP allocation state file (#1958) --- go.mod | 3 +- pkg/cri/cri.go | 39 +++++-- pkg/ipamd/datastore/data_store.go | 101 ++++++++++++------- pkg/ipamd/datastore/data_store_test.go | 134 +++++++++++++++++++++---- pkg/ipamd/ipamd_test.go | 17 +++- pkg/ipamd/rpc_handler.go | 6 +- 6 files changed, 230 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index 68a98952c6a..20fded293ad 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/coreos/go-iptables v0.4.5 github.com/golang/mock v1.4.1 github.com/golang/protobuf v1.4.3 + github.com/google/go-cmp v0.5.2 github.com/google/go-jsonnet v0.16.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 @@ -41,7 +42,6 @@ require ( github.com/go-logr/logr v0.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/google/go-cmp v0.5.2 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/googleapis/gnostic v0.5.1 // indirect @@ -66,6 +66,7 @@ require ( golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect golang.org/x/tools v0.1.5 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index 3c77b82b756..8940790865b 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -28,15 +28,29 @@ const ( dockerSocketPath = "unix:///var/run/dockershim.sock" ) +// PodSandboxMetadata contains metadata about pod sandboxes. +type PodSandboxMetadata struct { + // Pod's namespace + Namespace string + // Pod's name + Name string +} + // SandboxInfo provides container information type SandboxInfo struct { + // Pod sandbox's ID. ID string - IP string + // Pod sandbox's IP addresses + IPs []string + // Timestamp(in nanoSec) when sandbox was created + CreationTimestamp int64 + // Pod sandbox's metadata if any. + Metadata *PodSandboxMetadata } // APIs is the CRI interface type APIs interface { - GetRunningPodSandboxes(log logger.Logger) ([]*SandboxInfo, error) + GetRunningPodSandboxes(log logger.Logger) ([]SandboxInfo, error) } // Client is an empty struct @@ -48,7 +62,7 @@ func New() *Client { } //GetRunningPodSandboxes get running sandboxIDs -func (c *Client) GetRunningPodSandboxes(log logger.Logger) ([]*SandboxInfo, error) { +func (c *Client) GetRunningPodSandboxes(log logger.Logger) ([]SandboxInfo, error) { ctx := context.TODO() socketPath := dockerSocketPath @@ -76,7 +90,7 @@ func (c *Client) GetRunningPodSandboxes(log logger.Logger) ([]*SandboxInfo, erro return nil, err } - sandboxInfos := make([]*SandboxInfo, 0, len(sandboxes.GetItems())) + sandboxInfos := make([]SandboxInfo, 0, len(sandboxes.GetItems())) for _, sandbox := range sandboxes.GetItems() { status, err := client.PodSandboxStatus(ctx, &runtimeapi.PodSandboxStatusRequest{ PodSandboxId: sandbox.GetId(), @@ -99,14 +113,19 @@ func (c *Client) GetRunningPodSandboxes(log logger.Logger) ([]*SandboxInfo, erro for _, ip := range status.GetStatus().GetNetwork().GetAdditionalIps() { ips = append(ips, ip.GetIp()) } - - for _, ip := range ips { - info := SandboxInfo{ - ID: sandbox.GetId(), - IP: ip, + sandBoxInfo := SandboxInfo{ + ID: sandbox.GetId(), + IPs: ips, + CreationTimestamp: status.GetStatus().GetCreatedAt(), + } + if metadata := status.GetStatus().GetMetadata(); metadata != nil { + sandBoxInfo.Metadata = &PodSandboxMetadata{ + Namespace: metadata.GetNamespace(), + Name: metadata.GetName(), } - sandboxInfos = append(sandboxInfos, &info) } + + sandboxInfos = append(sandboxInfos, sandBoxInfo) } return sandboxInfos, nil } diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 3a78e3f0e00..3eacc0f3317 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -147,6 +147,12 @@ func (k IPAMKey) String() string { return fmt.Sprintf("%s/%s/%s", k.NetworkName, k.ContainerID, k.IfName) } +// IPAMMetadata is the metadata associated with IP allocations. +type IPAMMetadata struct { + K8SPodNamespace string `json:"k8sPodNamespace,omitempty"` + K8SPodName string `json:"k8sPodName,omitempty"` +} + // ENI represents a single ENI. Exported fields will be marshaled for introspection. type ENI struct { // AWS ENI ID @@ -170,8 +176,11 @@ type ENI struct { // AddressInfo contains information about an IP, Exported fields will be marshaled for introspection. type AddressInfo struct { + Address string + IPAMKey IPAMKey - Address string + IPAMMetadata IPAMMetadata + AssignedTime time.Time UnassignedTime time.Time } @@ -360,8 +369,10 @@ type CheckpointData struct { // in checkpoints. type CheckpointEntry struct { IPAMKey - IPv4 string `json:"ipv4,omitempty"` - IPv6 string `json:"ipv6,omitempty"` + IPv4 string `json:"ipv4,omitempty"` + IPv6 string `json:"ipv6,omitempty"` + AllocationTimestamp int64 `json:"allocationTimestamp"` + Metadata IPAMMetadata `json:"metadata"` } // ReadBackingStore initialises the IP allocation state from the @@ -383,21 +394,35 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { entries := make([]CheckpointEntry, 0, len(sandboxes)) for _, s := range sandboxes { ds.log.Debugf("Adding container ID: %v", s.ID) - if isv6Enabled { - ipv6Addr = s.IP - } else { - ipv4Addr = s.IP + + metadata := IPAMMetadata{} + // both containerd and dockershim populates the metadata, just be cautious to have this null check. + if s.Metadata != nil { + metadata.K8SPodNamespace = s.Metadata.Namespace + metadata.K8SPodName = s.Metadata.Name + } + + // note: ideally each sandbox should only contain one IP only, + // looping through them here is just to keep legacy code's behavior. + for _, ip := range s.IPs { + if isv6Enabled { + ipv6Addr = ip + } else { + ipv4Addr = ip + } + entries = append(entries, CheckpointEntry{ + // NB: These Backfill values are also assumed in UnassignPodIPAddress + IPAMKey: IPAMKey{ + NetworkName: backfillNetworkName, + ContainerID: s.ID, + IfName: backfillNetworkIface, + }, + IPv4: ipv4Addr, + IPv6: ipv6Addr, + AllocationTimestamp: s.CreationTimestamp, + Metadata: metadata, + }) } - entries = append(entries, CheckpointEntry{ - // NB: These Backfill values are also assumed in UnassignPodIPAddress - IPAMKey: IPAMKey{ - NetworkName: backfillNetworkName, - ContainerID: s.ID, - IfName: backfillNetworkIface, - }, - IPv4: ipv4Addr, - IPv6: ipv6Addr, - }) } data = CheckpointData{ Version: CheckpointFormatVersion, @@ -455,7 +480,7 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { } addr := &AddressInfo{Address: ipAddr.String()} cidr.IPAddresses[ipAddr.String()] = addr - ds.assignPodIPAddressUnsafe(allocation.IPAMKey, eni, addr) + ds.assignPodIPAddressUnsafe(addr, allocation.IPAMKey, allocation.Metadata, time.Unix(0, allocation.AllocationTimestamp)) ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) //Update prometheus for ips per cidr //Secondary IP mode will have /32:1 and Prefix mode will have /28: @@ -493,8 +518,10 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { for _, addr := range assignedAddr.IPAddresses { if addr.Assigned() { entry := CheckpointEntry{ - IPAMKey: addr.IPAMKey, - IPv4: addr.Address, + IPAMKey: addr.IPAMKey, + IPv4: addr.Address, + AllocationTimestamp: addr.AssignedTime.UnixNano(), + Metadata: addr.IPAMMetadata, } allocations = append(allocations, entry) } @@ -505,8 +532,10 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { for _, addr := range assignedAddr.IPAddresses { if addr.Assigned() { entry := CheckpointEntry{ - IPAMKey: addr.IPAMKey, - IPv6: addr.Address, + IPAMKey: addr.IPAMKey, + IPv6: addr.Address, + AllocationTimestamp: addr.AssignedTime.UnixNano(), + Metadata: addr.IPAMMetadata, } allocations = append(allocations, entry) } @@ -677,19 +706,19 @@ func (ds *DataStore) AddIPv6CidrToStore(eniID string, ipv6Cidr net.IPNet, isPref return nil } -func (ds *DataStore) AssignPodIPAddress(ipamKey IPAMKey, isIPv4Enabled bool, isIPv6Enabled bool) (ipv4Address string, +func (ds *DataStore) AssignPodIPAddress(ipamKey IPAMKey, ipamMetadata IPAMMetadata, isIPv4Enabled bool, isIPv6Enabled bool) (ipv4Address string, ipv6Address string, deviceNumber int, err error) { //Currently it's either v4 or v6. Dual Stack mode isn't supported. if isIPv4Enabled { - ipv4Address, deviceNumber, err = ds.AssignPodIPv4Address(ipamKey) + ipv4Address, deviceNumber, err = ds.AssignPodIPv4Address(ipamKey, ipamMetadata) } else if isIPv6Enabled { - ipv6Address, deviceNumber, err = ds.AssignPodIPv6Address(ipamKey) + ipv6Address, deviceNumber, err = ds.AssignPodIPv6Address(ipamKey, ipamMetadata) } return ipv4Address, ipv6Address, deviceNumber, err } // AssignPodIPv6Address assigns an IPv6 address to pod. Returns the assigned IPv6 address along with device number -func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey) (ipv6Address string, deviceNumber int, err error) { +func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey, ipamMetadata IPAMMetadata) (ipv6Address string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() @@ -723,7 +752,7 @@ func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey) (ipv6Address string, addr := &AddressInfo{Address: ipv6Address} V6Cidr.IPAddresses[ipv6Address] = addr - ds.assignPodIPAddressUnsafe(ipamKey, eni, addr) + ds.assignPodIPAddressUnsafe(addr, ipamKey, ipamMetadata, time.Now()) if err := ds.writeBackingStoreUnsafe(); err != nil { ds.log.Warnf("Failed to update backing store: %v", err) // Important! Unwind assignment @@ -740,7 +769,7 @@ func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey) (ipv6Address string, // AssignPodIPv4Address assigns an IPv4 address to pod // It returns the assigned IPv4 address, device number, error -func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, deviceNumber int, err error) { +func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMetadata) (ipv4address string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() @@ -785,7 +814,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, } availableCidr.IPAddresses[strPrivateIPv4] = addr - ds.assignPodIPAddressUnsafe(ipamKey, eni, addr) + ds.assignPodIPAddressUnsafe(addr, ipamKey, ipamMetadata, time.Now()) if err := ds.writeBackingStoreUnsafe(); err != nil { ds.log.Warnf("Failed to update backing store: %v", err) @@ -806,8 +835,8 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, return "", -1, errors.New("assignPodIPv4AddressUnsafe: no available IP/Prefix addresses") } -// It returns the assigned IPv4 address, device number -func (ds *DataStore) assignPodIPAddressUnsafe(ipamKey IPAMKey, eni *ENI, addr *AddressInfo) (string, int) { +// assignPodIPAddressUnsafe mark Address as assigned. +func (ds *DataStore) assignPodIPAddressUnsafe(addr *AddressInfo, ipamKey IPAMKey, ipamMetadata IPAMMetadata, assignedTime time.Time) { ds.log.Infof("AssignPodIPv4Address: Assign IP %v to sandbox %s", addr.Address, ipamKey) @@ -815,14 +844,15 @@ func (ds *DataStore) assignPodIPAddressUnsafe(ipamKey IPAMKey, eni *ENI, addr *A panic("addr already assigned") } addr.IPAMKey = ipamKey // This marks the addr as assigned + addr.IPAMMetadata = ipamMetadata + addr.AssignedTime = assignedTime ds.assigned++ // Prometheus gauge assignedIPs.Set(float64(ds.assigned)) - - return addr.Address, eni.DeviceNumber } +// unassignPodIPAddressUnsafe mark Address as unassigned. func (ds *DataStore) unassignPodIPAddressUnsafe(addr *AddressInfo) { if !addr.Assigned() { // Already unassigned @@ -831,6 +861,7 @@ func (ds *DataStore) unassignPodIPAddressUnsafe(addr *AddressInfo) { ds.log.Infof("UnAssignPodIPAddress: Unassign IP %v from sandbox %s", addr.Address, addr.IPAMKey) addr.IPAMKey = IPAMKey{} // unassign the addr + addr.IPAMMetadata = IPAMMetadata{} ds.assigned-- // Prometheus gauge assignedIPs.Set(float64(ds.assigned)) @@ -1179,10 +1210,12 @@ func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, d return nil, "", 0, ErrUnknownPod } + originalIPAMMetadata := addr.IPAMMetadata + originalAssignedTime := addr.AssignedTime ds.unassignPodIPAddressUnsafe(addr) if err := ds.writeBackingStoreUnsafe(); err != nil { // Unwind un-assignment - ds.assignPodIPAddressUnsafe(ipamKey, eni, addr) + ds.assignPodIPAddressUnsafe(addr, ipamKey, originalIPAMMetadata, originalAssignedTime) return nil, "", 0, err } addr.UnassignedTime = time.Now() diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index aeac3f358c5..3658150458f 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -19,6 +19,9 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/stretchr/testify/assert" @@ -80,7 +83,9 @@ func TestDeleteENI(t *testing.T) { ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) assert.NoError(t, err) - ip, device, err := ds.AssignPodIPv4Address(IPAMKey{"net1", "sandbox1", "eth0"}) + ip, device, err := ds.AssignPodIPv4Address( + IPAMKey{"net1", "sandbox1", "eth0"}, + IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, device) @@ -185,7 +190,7 @@ func TestDelENIIPv4Address(t *testing.T) { // Assign a pod. key := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, device, err := ds.AssignPodIPv4Address(key) + ip, device, err := ds.AssignPodIPv4Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, device) @@ -234,6 +239,13 @@ func TestPodIPv4Address(t *testing.T) { checkpoint := NewTestCheckpoint(struct{}{}) ds := NewDataStore(Testlog, checkpoint, false) + checkpointDataCmpOpts := cmp.Options{ + cmpopts.IgnoreFields(CheckpointEntry{}, "AllocationTimestamp"), + cmpopts.SortSlices(func(lhs CheckpointEntry, rhs CheckpointEntry) bool { + return lhs.ContainerID < rhs.ContainerID + }), + } + err := ds.AddENI("eni-1", 1, true, false, false) assert.NoError(t, err) @@ -245,19 +257,28 @@ func TestPodIPv4Address(t *testing.T) { assert.NoError(t, err) key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, _, err := ds.AssignPodIPv4Address(key1) + ip, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, ds.total) assert.Equal(t, 1, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs)) assert.Equal(t, 1, ds.eniPool["eni-1"].AssignedIPv4Addresses()) - assert.Equal(t, checkpoint.Data, &CheckpointData{ + + expectedCheckpointData := &CheckpointData{ Version: CheckpointFormatVersion, Allocations: []CheckpointEntry{ - {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPv4: "1.1.1.1", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + }, }, - }) + } + assert.True(t, + cmp.Equal(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + cmp.Diff(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + ) podsInfos := ds.AllocatedIPs() assert.Equal(t, len(podsInfos), 1) @@ -267,7 +288,7 @@ func TestPodIPv4Address(t *testing.T) { assert.NoError(t, err) // duplicate add - ip, _, err = ds.AssignPodIPv4Address(key1) // same id + ip, _, err = ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) // same id assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.1") assert.Equal(t, ds.total, 2) @@ -280,24 +301,52 @@ func TestPodIPv4Address(t *testing.T) { // Checkpoint error checkpoint.Error = errors.New("fake checkpoint error") key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2) + _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.Error(t, err) - assert.Equal(t, checkpoint.Data, &CheckpointData{ + + expectedCheckpointData = &CheckpointData{ Version: CheckpointFormatVersion, Allocations: []CheckpointEntry{ - {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPv4: "1.1.1.1", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + }, }, - }) + } + assert.True(t, + cmp.Equal(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + cmp.Diff(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + ) checkpoint.Error = nil - ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2) + ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, ip, "1.1.2.2") assert.Equal(t, ds.total, 2) assert.Equal(t, ds.assigned, 2) assert.Equal(t, len(ds.eniPool["eni-2"].AvailableIPv4Cidrs), 1) assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 1) - assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 2) + + expectedCheckpointData = &CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: []CheckpointEntry{ + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPv4: "1.1.1.1", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + }, + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, + IPv4: "1.1.2.2", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, + }, + }, + } + assert.True(t, + cmp.Equal(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + cmp.Diff(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + ) podsInfos = ds.AllocatedIPs() assert.Equal(t, len(podsInfos), 2) @@ -307,18 +356,41 @@ func TestPodIPv4Address(t *testing.T) { assert.NoError(t, err) key3 := IPAMKey{"net0", "sandbox-3", "eth0"} - ip, _, err = ds.AssignPodIPv4Address(key3) + ip, _, err = ds.AssignPodIPv4Address(key3, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.2") assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 3) assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 2) assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 2) - assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 3) + expectedCheckpointData = &CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: []CheckpointEntry{ + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPv4: "1.1.1.1", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + }, + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, + IPv4: "1.1.2.2", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, + }, + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, + IPv4: "1.1.1.2", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, + }, + }, + } + assert.True(t, + cmp.Equal(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + cmp.Diff(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + ) // no more IP addresses key4 := IPAMKey{"net0", "sandbox-4", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key4) + _, _, err = ds.AssignPodIPv4Address(key4, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-4"}) assert.Error(t, err) // Unassign unknown Pod _, _, _, err = ds.UnassignPodIPAddress(key4) @@ -331,7 +403,25 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, deviceNum, pod1Ns2Device) assert.Equal(t, len(ds.eniPool["eni-2"].AvailableIPv4Cidrs), 1) assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 0) - assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 2) + expectedCheckpointData = &CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: []CheckpointEntry{ + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPv4: "1.1.1.1", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + }, + { + IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, + IPv4: "1.1.1.2", + Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, + }, + }, + } + assert.True(t, + cmp.Equal(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + cmp.Diff(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts), + ) noWarmIPTarget := 0 noMinimumIPTarget := 0 @@ -358,13 +448,13 @@ func TestGetIPStatsV4(t *testing.T) { ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - _, _, err := ds.AssignPodIPv4Address(key1) + _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2) + _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, @@ -407,7 +497,7 @@ func TestGetIPStatsV6(t *testing.T) { ipv6Addr := net.IPNet{IP: net.IP{0x21, 0xdb, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, Mask: net.CIDRMask(80, 128)} _ = v6ds.AddIPv6CidrToStore("eni-1", ipv6Addr, true) key3 := IPAMKey{"netv6", "sandbox-3", "eth0"} - _, _, err := v6ds.AssignPodIPv6Address(key3) + _, _, err := v6ds.AssignPodIPv6Address(key3, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) assert.NoError(t, err) assert.Equal(t, @@ -431,13 +521,13 @@ func TestWarmENIInteractions(t *testing.T) { ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - _, _, err := ds.AssignPodIPv4Address(key1) + _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2) + _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.2.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index db119e2ad21..82bd35516be 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -1216,6 +1216,9 @@ func datastoreWith1Pod1() *datastore.DataStore { NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0", + }, datastore.IPAMMetadata{ + K8SPodNamespace: "default", + K8SPodName: "sample-pod", }) return datastoreWith1Pod1 } @@ -1229,7 +1232,10 @@ func datastoreWith3Pods() *datastore.DataStore { ContainerID: fmt.Sprintf("sandbox-%d", i), IfName: "eth0", } - _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key) + _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key, datastore.IPAMMetadata{ + K8SPodNamespace: "default", + K8SPodName: fmt.Sprintf("sample-pod-%d", i), + }) } return datastoreWith3Pods } @@ -1249,6 +1255,9 @@ func datastoreWith1Pod1FromPrefix() *datastore.DataStore { NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0", + }, datastore.IPAMMetadata{ + K8SPodNamespace: "default", + K8SPodName: "sample-pod", }) return datastoreWith1Pod1 } @@ -1262,7 +1271,11 @@ func datastoreWith3PodsFromPrefix() *datastore.DataStore { ContainerID: fmt.Sprintf("sandbox-%d", i), IfName: "eth0", } - _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key) + _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key, + datastore.IPAMMetadata{ + K8SPodNamespace: "default", + K8SPodName: fmt.Sprintf("sample-pod-%d", i), + }) } return datastoreWith3Pods } diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index cb6b8503b95..c97074b8d20 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -144,7 +144,11 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp IfName: in.IfName, NetworkName: in.NetworkName, } - ipv4Addr, ipv6Addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPAddress(ipamKey, s.ipamContext.enableIPv4, s.ipamContext.enableIPv6) + ipamMetadata := datastore.IPAMMetadata{ + K8SPodNamespace: in.K8S_POD_NAMESPACE, + K8SPodName: in.K8S_POD_NAME, + } + ipv4Addr, ipv6Addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPAddress(ipamKey, ipamMetadata, s.ipamContext.enableIPv4, s.ipamContext.enableIPv6) } var pbVPCV4cidrs, pbVPCV6cidrs []string