agent: add interface memHotplugByProbe

we need to notify guest kernel about memory hot-added event via probe interface.
hot-added memory deivce should be sliced into the size of memory section.

Fixes: kata-containers#1149

Signed-off-by: Penny Zheng <[email protected]>

virtcontainers/agent.go

@@ -216,6 +216,12 @@ type agent interface {
 	// cpuOnly specifies that we should online cpu or online memory or both
 	onlineCPUMem(cpus uint32, cpuOnly bool) error
 
+	// memHotplugByProbe will notify the guest kernel about memory hotplug event through
+	// probe interface.
+	// This function should be called after hot adding Memory and before online memory.
+	// addr specifies the address of the recently hotplugged or unhotplugged memory device.
+	memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error
+
 	// statsContainer will tell the agent to get stats from a container related to a Sandbox
 	statsContainer(sandbox *Sandbox, c Container) (*ContainerStats, error)
 

virtcontainers/hyperstart_agent.go

@@ -891,6 +891,11 @@ func (h *hyper) sendCmd(proxyCmd hyperstartProxyCmd) (interface{}, error) {
 	return h.client.HyperWithTokens(proxyCmd.cmd, tokens, proxyCmd.message)
 }
 
+func (h *hyper) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
+	// hyperstart-agent does not support notify memory hotplug event via probe interface
+	return nil
+}
+
 func (h *hyper) onlineCPUMem(cpus uint32, cpuOnly bool) error {
 	// hyperstart-agent uses udev to online CPUs automatically
 	return nil

virtcontainers/kata_agent.go

@@ -1346,6 +1346,30 @@ func (k *kataAgent) resumeContainer(sandbox *Sandbox, c Container) error {
 	return err
 }
 
+func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
+	if memorySectionSizeMB == uint32(0) {
+		return fmt.Errorf("memorySectionSizeMB couldn't be zero")
+	}
+	// hot-added memory device should be sliced into the size of memory section, which is the basic unit for
+	// memory hotplug
+	numSection := uint64(sizeMB / memorySectionSizeMB)
+	var addrList []uint64
+	index := uint64(0)
+	for index < numSection {
+		k.Logger().WithFields(logrus.Fields{
+			"addr": fmt.Sprintf("0x%x", addr+(index*uint64(memorySectionSizeMB))<<20),
+		}).Debugf("notify guest kernel the address of memory device")
+		addrList = append(addrList, addr+(index*uint64(memorySectionSizeMB))<<20)
+		index++
+	}
+	req := &grpc.MemHotplugByProbeRequest{
+		MemHotplugProbeAddr: addrList,
+	}
+
+	_, err := k.sendReq(req)
+	return err
+}
+
 func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
 	req := &grpc.OnlineCPUMemRequest{
 		Wait:    false,
@@ -1574,6 +1598,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers["grpc.GuestDetailsRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.GetGuestDetails(ctx, req.(*grpc.GuestDetailsRequest), opts...)
 	}
+	k.reqHandlers["grpc.MemHotplugByProbeRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.MemHotplugByProbe(ctx, req.(*grpc.MemHotplugByProbeRequest), opts...)
+	}
 	k.reqHandlers["grpc.CopyFileRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.CopyFile(ctx, req.(*grpc.CopyFileRequest), opts...)
 	}

virtcontainers/noop_agent.go

@@ -91,6 +91,11 @@ func (n *noopAgent) updateContainer(sandbox *Sandbox, c Container, resources spe
 	return nil
 }
 
+// memHotplugByProbe is the Noop agent notify meomory hotplug event via probe interface implementation. It does nothing.
+func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
+	return nil
+}
+
 // onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
 func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
 	return nil

virtcontainers/sandbox.go

@@ -1644,11 +1644,18 @@ func (s *Sandbox) updateResources() error {
 
 	// Update Memory
 	s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
-	newMemory, _, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
+	newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
 	if err != nil {
 		return err
 	}
 	s.Logger().Debugf("Sandbox memory size: %d Byte", newMemory)
+	if s.state.GuestMemoryHotplugProbe && updatedMemoryDevice.addr != 0 {
+		//notify the guest kernel about memory hot-add event, before onlining them
+		s.Logger().Debugf("notify guest kernel memory hot-add event via probe interface, memory device located at 0x%x", updatedMemoryDevice.addr)
+		if err := s.agent.memHotplugByProbe(updatedMemoryDevice.addr, uint32(updatedMemoryDevice.sizeMB), s.state.GuestMemoryBlockSizeMB); err != nil {
+			return err
+		}
+	}
 	if err := s.agent.onlineCPUMem(0, false); err != nil {
 		return err
 	}