agent: add interface memHotplugByProbe for memory hotplug via probe i…

…nterfacce we need to notify guest kernel about memory hot-added event via probe interface. hot-added memory deivce should be sliced into the size of memory section. Fixes: kata-containers#1149 Signed-off-by: Penny Zheng <[email protected]>
Pennyzct · Mar 6, 2019 · 9a8c2fe · 9a8c2fe
1 parent 62a4d54
commit 9a8c2fe
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 1 deletion.
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
@@ -213,6 +213,12 @@ type agent interface {
 	// cpuOnly specifies that we should online cpu or online memory or both
 	onlineCPUMem(cpus uint32, cpuOnly bool) error
 
+	// memHotplugByProbe will notify the guest kernel about memory hotplug event through
+	// probe interface.
+	// This function should be called after hot adding Memory and before online memory.
+	// addr specifies the address of the recently hotplugged or unhotplugged memory device.
+	memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error
+
 	// statsContainer will tell the agent to get stats from a container related to a Sandbox
 	statsContainer(sandbox *Sandbox, c Container) (*ContainerStats, error)
 

diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go
@@ -887,6 +887,11 @@ func (h *hyper) sendCmd(proxyCmd hyperstartProxyCmd) (interface{}, error) {
 	return h.client.HyperWithTokens(proxyCmd.cmd, tokens, proxyCmd.message)
 }
 
+func (h *hyper) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
+	// hyperstart-agent does not support notify memory hotplug event via probe interface
+	return nil
+}
+
 func (h *hyper) onlineCPUMem(cpus uint32, cpuOnly bool) error {
 	// hyperstart-agent uses udev to online CPUs automatically
 	return nil

diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
@@ -1328,6 +1328,27 @@ func (k *kataAgent) resumeContainer(sandbox *Sandbox, c Container) error {
 	return err
 }
 
+func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
+	// hot-added memory device should be sliced into the size of memory section, which is the basic unit for
+	// memory hotplug
+	numSection := uint64(sizeMB / memorySectionSizeMB)
+	var addrList []uint64
+	index := uint64(0)
+	for index < numSection {
+		k.Logger().WithFields(logrus.Fields{
+			"addr": fmt.Sprintf("0x%x", addr+(index*uint64(memorySectionSizeMB))<<20),
+		}).Debugf("notify guest kernel the address of memory device")
+		addrList = append(addrList, addr+(index*uint64(memorySectionSizeMB))<<20)
+		index++
+	}
+	req := &grpc.MemHotplugByProbeRequest{
+		MemHotplugProbeAddr: addrList,
+	}
+
+	_, err := k.sendReq(req)
+	return err
+}
+
 func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
 	req := &grpc.OnlineCPUMemRequest{
 		Wait:    false,
@@ -1556,6 +1577,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers["grpc.GuestDetailsRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.GetGuestDetails(ctx, req.(*grpc.GuestDetailsRequest), opts...)
 	}
+	k.reqHandlers["grpc.MemHotplugByProbeRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.MemHotplugByProbe(ctx, req.(*grpc.MemHotplugByProbeRequest), opts...)
+	}
 	k.reqHandlers["grpc.CopyFileRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.CopyFile(ctx, req.(*grpc.CopyFileRequest), opts...)
 	}

diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
@@ -91,6 +91,11 @@ func (n *noopAgent) updateContainer(sandbox *Sandbox, c Container, resources spe
 	return nil
 }
 
+// memHotplugByProbe is the Noop agent notify meomory hotplug event via probe interface implementation. It does nothing.
+func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
+	return nil
+}
+
 // onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
 func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
 	return nil

diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
@@ -1658,11 +1658,18 @@ func (s *Sandbox) updateResources() error {
 
 	// Update Memory
 	s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
-	newMemory, _, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
+	newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
 	if err != nil {
 		return err
 	}
 	s.Logger().Debugf("Sandbox memory size: %d Byte", newMemory)
+	if s.state.GuestMemoryHotplugProbe && updatedMemoryDevice.addr != 0 {
+		//notify the guest kernel about memory hot-add event, before onlining them
+		s.Logger().Debugf("notify guest kernel memory hot-add event via probe interface, memory device located at 0x%x", updatedMemoryDevice.addr)
+		if err := s.agent.memHotplugByProbe(updatedMemoryDevice.addr, uint32(updatedMemoryDevice.sizeMB), s.state.GuestMemoryBlockSizeMB); err != nil {
+			return err
+		}
+	}
 	if err := s.agent.onlineCPUMem(0, false); err != nil {
 		return err
 	}