From 450ec9610bbed65424278566026b0758859f276a Mon Sep 17 00:00:00 2001 From: Penny Zheng Date: Fri, 11 Jan 2019 14:31:08 +0800 Subject: [PATCH] agent: add interface memHotplugByProbe we need to notify guest kernel about memory hot-added event via probe interface. hot-added memory deivce should be sliced into the size of memory section. Fixes: #1149 Signed-off-by: Penny Zheng --- virtcontainers/agent.go | 6 ++++++ virtcontainers/hyperstart_agent.go | 5 +++++ virtcontainers/kata_agent.go | 27 +++++++++++++++++++++++++++ virtcontainers/noop_agent.go | 5 +++++ virtcontainers/sandbox.go | 9 ++++++++- 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go index 804f47b789..8b94f3023b 100644 --- a/virtcontainers/agent.go +++ b/virtcontainers/agent.go @@ -216,6 +216,12 @@ type agent interface { // cpuOnly specifies that we should online cpu or online memory or both onlineCPUMem(cpus uint32, cpuOnly bool) error + // memHotplugByProbe will notify the guest kernel about memory hotplug event through + // probe interface. + // This function should be called after hot adding Memory and before online memory. + // addr specifies the address of the recently hotplugged or unhotplugged memory device. + memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error + // statsContainer will tell the agent to get stats from a container related to a Sandbox statsContainer(sandbox *Sandbox, c Container) (*ContainerStats, error) diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index dc9f3aed40..e1372c4b57 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -891,6 +891,11 @@ func (h *hyper) sendCmd(proxyCmd hyperstartProxyCmd) (interface{}, error) { return h.client.HyperWithTokens(proxyCmd.cmd, tokens, proxyCmd.message) } +func (h *hyper) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + // hyperstart-agent does not support notify memory hotplug event via probe interface + return nil +} + func (h *hyper) onlineCPUMem(cpus uint32, cpuOnly bool) error { // hyperstart-agent uses udev to online CPUs automatically return nil diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 84352d84b7..780ac75df0 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -1346,6 +1346,30 @@ func (k *kataAgent) resumeContainer(sandbox *Sandbox, c Container) error { return err } +func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + if memorySectionSizeMB == uint32(0) { + return fmt.Errorf("memorySectionSizeMB couldn't be zero") + } + // hot-added memory device should be sliced into the size of memory section, which is the basic unit for + // memory hotplug + numSection := uint64(sizeMB / memorySectionSizeMB) + var addrList []uint64 + index := uint64(0) + for index < numSection { + k.Logger().WithFields(logrus.Fields{ + "addr": fmt.Sprintf("0x%x", addr+(index*uint64(memorySectionSizeMB))<<20), + }).Debugf("notify guest kernel the address of memory device") + addrList = append(addrList, addr+(index*uint64(memorySectionSizeMB))<<20) + index++ + } + req := &grpc.MemHotplugByProbeRequest{ + MemHotplugProbeAddr: addrList, + } + + _, err := k.sendReq(req) + return err +} + func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { req := &grpc.OnlineCPUMemRequest{ Wait: false, @@ -1574,6 +1598,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers["grpc.GuestDetailsRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.GetGuestDetails(ctx, req.(*grpc.GuestDetailsRequest), opts...) } + k.reqHandlers["grpc.MemHotplugByProbeRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.MemHotplugByProbe(ctx, req.(*grpc.MemHotplugByProbeRequest), opts...) + } k.reqHandlers["grpc.CopyFileRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.CopyFile(ctx, req.(*grpc.CopyFileRequest), opts...) } diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go index db9a3dcbdf..cf609128f2 100644 --- a/virtcontainers/noop_agent.go +++ b/virtcontainers/noop_agent.go @@ -91,6 +91,11 @@ func (n *noopAgent) updateContainer(sandbox *Sandbox, c Container, resources spe return nil } +// memHotplugByProbe is the Noop agent notify meomory hotplug event via probe interface implementation. It does nothing. +func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + return nil +} + // onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing. func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { return nil diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index be4a9cb8c2..500bb41fbd 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1644,11 +1644,18 @@ func (s *Sandbox) updateResources() error { // Update Memory s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") - newMemory, _, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) + newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) if err != nil { return err } s.Logger().Debugf("Sandbox memory size: %d Byte", newMemory) + if s.state.GuestMemoryHotplugProbe && updatedMemoryDevice.addr != 0 { + //notify the guest kernel about memory hot-add event, before onlining them + s.Logger().Debugf("notify guest kernel memory hot-add event via probe interface, memory device located at 0x%x", updatedMemoryDevice.addr) + if err := s.agent.memHotplugByProbe(updatedMemoryDevice.addr, uint32(updatedMemoryDevice.sizeMB), s.state.GuestMemoryBlockSizeMB); err != nil { + return err + } + } if err := s.agent.onlineCPUMem(0, false); err != nil { return err }