diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go index cf0c71c504..73f0edfd3c 100644 --- a/virtcontainers/agent.go +++ b/virtcontainers/agent.go @@ -213,6 +213,12 @@ type agent interface { // cpuOnly specifies that we should online cpu or online memory or both onlineCPUMem(cpus uint32, cpuOnly bool) error + // memHotplugByProbe will notify the guest kernel about memory hotplug event through + // probe interface. + // This function should be called after hot adding Memory and before online memory. + // addr specifies the address of the recently hotplugged or unhotplugged memory device. + memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error + // statsContainer will tell the agent to get stats from a container related to a Sandbox statsContainer(sandbox *Sandbox, c Container) (*ContainerStats, error) diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index feabc1f5a2..80bafa9ec4 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -886,6 +886,11 @@ func (h *hyper) sendCmd(proxyCmd hyperstartProxyCmd) (interface{}, error) { return h.client.HyperWithTokens(proxyCmd.cmd, tokens, proxyCmd.message) } +func (h *hyper) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + // hyperstart-agent does not support notify memory hotplug event via probe interface + return nil +} + func (h *hyper) onlineCPUMem(cpus uint32, cpuOnly bool) error { // hyperstart-agent uses udev to online CPUs automatically return nil diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 3f0abc423f..11b55c8758 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -1327,6 +1327,24 @@ func (k *kataAgent) resumeContainer(sandbox *Sandbox, c Container) error { return err } +func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) (error) { + // hot-added memory device should be sliced into the size of memory section, which is the basic unit for + // memory hotplug + numSection := sizeMB/memorySectionSizeMB + var addrList []uint64 + index := uint32(0) + for index < numSection { + addrList = append(addrList, addr + uint64((index * memorySectionSizeMB) << 20)) + index++ + } + req := &grpc.MemHotplugByProbeRequest{ + MemHotplugProbeAddr: addrList, + } + + _, err := k.sendReq(req) + return err +} + func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { req := &grpc.OnlineCPUMemRequest{ Wait: false, @@ -1555,6 +1573,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers["grpc.GuestDetailsRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.GetGuestDetails(ctx, req.(*grpc.GuestDetailsRequest), opts...) } + k.reqHandlers["grpc.MemHotplugByProbeRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.MemHotplugByProbe(ctx, req.(*grpc.MemHotplugByProbeRequest), opts...) + } k.reqHandlers["grpc.CopyFileRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.CopyFile(ctx, req.(*grpc.CopyFileRequest), opts...) } diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go index 1783454b93..432b9ecf5d 100644 --- a/virtcontainers/noop_agent.go +++ b/virtcontainers/noop_agent.go @@ -91,6 +91,11 @@ func (n *noopAgent) updateContainer(sandbox *Sandbox, c Container, resources spe return nil } +// memHotplugByProbe is the Noop agent notify meomory hotplug event via probe interface implementation. It does nothing. +func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + return nil +} + // onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing. func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { return nil diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 7db519caec..0e81de07c9 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1704,11 +1704,18 @@ func (s *Sandbox) updateResources() error { // Update Memory s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") - newMemory, _, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) + newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) if err != nil { return err } s.Logger().Debugf("Sandbox memory size: %d Byte", newMemory) + if s.state.GuestMemoryHotplugProbe && updatedMemoryDevice.addr != 0 { + //notify the guest kernel about memory hot-add event, before onlining them + s.Logger().Debugf("notify guest kernel memory hot-add event via probe interface, memory device located at 0x%x", updatedMemoryDevice.addr) + if err := s.agent.memHotplugByProbe(updatedMemoryDevice.addr, uint32(updatedMemoryDevice.sizeMB), s.state.GuestMemoryBlockSizeMB); err != nil { + return err + } + } if err := s.agent.onlineCPUMem(0, false); err != nil { return err }