diff --git a/virtcontainers/container.go b/virtcontainers/container.go index a9eba63a5b..94cc8d92f8 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -253,8 +253,18 @@ type SystemMountsInfo struct { type ContainerDevice struct { // ID is device id referencing the device from sandbox's device manager ID string + // ContainerPath is device path displayed in container ContainerPath string + + // FileMode permission bits for the device. + FileMode os.FileMode + + // UID is user ID in the container namespace + UID uint32 + + // GID is group ID in the container namespace + GID uint32 } // Container is composed of a set of containers and a runtime environment. @@ -680,6 +690,9 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err storedDevices = append(storedDevices, ContainerDevice{ ID: dev.DeviceID(), ContainerPath: info.ContainerPath, + FileMode: info.FileMode, + UID: info.UID, + GID: info.GID, }) } c.devices = filterDevices(sandbox, c, storedDevices) @@ -749,6 +762,8 @@ func (c *Container) create() (err error) { return } + c.sandbox.PersistDevices() + process, err := c.sandbox.agent.createContainer(c.sandbox, c) if err != nil { return err diff --git a/virtcontainers/device/api/interface.go b/virtcontainers/device/api/interface.go index da15831aef..560a40d1b3 100644 --- a/virtcontainers/device/api/interface.go +++ b/virtcontainers/device/api/interface.go @@ -10,6 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) var devLogger = logrus.WithField("subsystem", "device") @@ -63,6 +64,9 @@ type Device interface { Reference() uint // Dereference removes one reference to device then returns final ref count Dereference() uint + + // Persist convert and return data in persist format + Dump() persistapi.DeviceState } // DeviceManager can be used to create a new device, this can be used as single diff --git a/virtcontainers/device/drivers/block.go b/virtcontainers/device/drivers/block.go index b977e30e8a..d45782fb8c 100644 --- a/virtcontainers/device/drivers/block.go +++ b/virtcontainers/device/drivers/block.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -142,5 +143,25 @@ func (device *BlockDevice) GetDeviceInfo() interface{} { return device.BlockDrive } +// Persist convert and return data in persist format +func (device *BlockDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + + drive := device.BlockDrive + ds.BlockDrive = &persistapi.BlockDrive{ + File: drive.File, + Format: drive.Format, + ID: drive.ID, + Index: drive.Index, + MmioAddr: drive.MmioAddr, + PCIAddr: drive.PCIAddr, + SCSIAddr: drive.SCSIAddr, + NvdimmID: drive.NvdimmID, + VirtPath: drive.VirtPath, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/generic.go b/virtcontainers/device/drivers/generic.go index 10835e3f16..be700fbc75 100644 --- a/virtcontainers/device/drivers/generic.go +++ b/virtcontainers/device/drivers/generic.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) // GenericDevice refers to a device that is neither a VFIO device or block device. @@ -127,3 +128,19 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error) } } } + +// Persist convert and return data in persist format +func (device *GenericDevice) Dump() persistapi.DeviceState { + info := device.DeviceInfo + return persistapi.DeviceState{ + ID: device.ID, + Type: string(device.DeviceType()), + RefCount: device.RefCount, + AttachCount: device.AttachCount, + + DevType: info.DevType, + Major: info.Major, + Minor: info.Minor, + DriverOptions: info.DriverOptions, + } +} diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go index a0a80dec72..96af2bfa4c 100644 --- a/virtcontainers/device/drivers/vfio.go +++ b/virtcontainers/device/drivers/vfio.go @@ -17,6 +17,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -129,6 +130,23 @@ func (device *VFIODevice) GetDeviceInfo() interface{} { return device.VfioDevs } +// Persist convert and return data in persist format +func (device *VFIODevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + + devs := device.VfioDevs + for _, dev := range devs { + ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{ + ID: dev.ID, + Type: string(dev.Type), + BDF: dev.BDF, + SysfsDev: dev.SysfsDev, + }) + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/vhost_user_blk.go b/virtcontainers/device/drivers/vhost_user_blk.go index cfc53d4145..87eb40d61e 100644 --- a/virtcontainers/device/drivers/vhost_user_blk.go +++ b/virtcontainers/device/drivers/vhost_user_blk.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -79,5 +80,18 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} { return &device.VhostUserDeviceAttrs } +// Persist convert and return data in persist format +func (device *VhostUserBlkDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ + DevID: device.DevID, + SocketPath: device.SocketPath, + Type: string(device.Type), + MacAddress: device.MacAddress, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/vhost_user_net.go b/virtcontainers/device/drivers/vhost_user_net.go index 2ab31dd620..08dca9269e 100644 --- a/virtcontainers/device/drivers/vhost_user_net.go +++ b/virtcontainers/device/drivers/vhost_user_net.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -79,5 +80,18 @@ func (device *VhostUserNetDevice) GetDeviceInfo() interface{} { return &device.VhostUserDeviceAttrs } +// Persist convert and return data in persist format +func (device *VhostUserNetDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ + DevID: device.DevID, + SocketPath: device.SocketPath, + Type: string(device.Type), + MacAddress: device.MacAddress, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/device/drivers/vhost_user_scsi.go b/virtcontainers/device/drivers/vhost_user_scsi.go index d34c50ec04..0622a49148 100644 --- a/virtcontainers/device/drivers/vhost_user_scsi.go +++ b/virtcontainers/device/drivers/vhost_user_scsi.go @@ -11,6 +11,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -78,5 +79,18 @@ func (device *VhostUserSCSIDevice) GetDeviceInfo() interface{} { return &device.VhostUserDeviceAttrs } +// Persist convert and return data in persist format +func (device *VhostUserSCSIDevice) Dump() persistapi.DeviceState { + ds := device.GenericDevice.Dump() + ds.Type = string(device.DeviceType()) + ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ + DevID: device.DevID, + SocketPath: device.SocketPath, + Type: string(device.Type), + MacAddress: device.MacAddress, + } + return ds +} + // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index e6c2bfdf2b..4ff49a204f 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -10,6 +10,7 @@ import ( //"github.com/sirupsen/logrus" + "github.com/kata-containers/runtime/virtcontainers/device/api" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) @@ -28,14 +29,17 @@ func (s *Sandbox) PersistState() { ss.GuestMemoryBlockSizeMB = s.state.GuestMemoryBlockSizeMB for id, cont := range s.containers { - cs[id] = persistapi.ContainerState{ - State: string(cont.state.State), - Rootfs: persistapi.RootfsState{ - BlockDeviceID: cont.state.BlockDeviceID, - FsType: cont.state.Fstype, - }, - ShimPid: cont.state.Pid, + state := persistapi.ContainerState{} + if v, ok := cs[id]; ok { + state = v + } + state.State = string(cont.state.State) + state.Rootfs = persistapi.RootfsState{ + BlockDeviceID: cont.state.BlockDeviceID, + FsType: cont.state.Fstype, } + state.ShimPid = cont.state.Pid + cs[id] = state } return nil }) @@ -49,6 +53,41 @@ func (s *Sandbox) PersistHvState() { }) } +func deviceToDeviceState(devices []api.Device) (dss []persistapi.DeviceState) { + for _, dev := range devices { + dss = append(dss, dev.Dump()) + } + return +} + +// PersistDevices register hook to save device informations +func (s *Sandbox) PersistDevices() { + s.store.RegisterHook("devices", func(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error { + ss.Devices = deviceToDeviceState(s.devManager.GetAllDevices()) + + for id, cont := range s.containers { + state := persistapi.ContainerState{} + if v, ok := cs[id]; ok { + state = v + } + + state.DeviceMaps = nil + for _, dev := range cont.devices { + state.DeviceMaps = append(state.DeviceMaps, persistapi.DeviceMap{ + ID: dev.ID, + ContainerPath: dev.ContainerPath, + FileMode: dev.FileMode, + UID: dev.UID, + GID: dev.GID, + }) + } + + cs[id] = state + } + return nil + }) +} + // Restore will restore data from persist disk on disk func (s *Sandbox) Restore() error { if err := s.store.Restore(s.id); err != nil { diff --git a/virtcontainers/persist/api/device.go b/virtcontainers/persist/api/device.go index 2b30d14cf7..6460fcbd7e 100644 --- a/virtcontainers/persist/api/device.go +++ b/virtcontainers/persist/api/device.go @@ -23,6 +23,9 @@ type BlockDrive struct { // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index Index int + // MmioAddr is used to identify the slot at which the drive is attached (order?). + MmioAddr string + // PCIAddr is the PCI address used to identify the slot at which the drive is attached. PCIAddr string @@ -30,6 +33,9 @@ type BlockDrive struct { // SCSI address is in the format SCSI-Id:LUN SCSIAddr string + // NvdimmID is the nvdimm id inside the VM + NvdimmID string + // VirtPath at which the device appears inside the VM, outside of the container mount namespace VirtPath string } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 75e5374f82..5469d28d4e 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -2009,6 +2009,7 @@ func (s *Sandbox) AddDevice(info config.DeviceInfo) (api.Device, error) { return nil, err } + s.PersistDevices() return b, nil }