persist: persist device data

Persist device information to relative file

Signed-off-by: Wei Zhang <[email protected]>

virtcontainers/container.go

@@ -253,8 +253,18 @@ type SystemMountsInfo struct {
 type ContainerDevice struct {
 	// ID is device id referencing the device from sandbox's device manager
 	ID string
+
 	// ContainerPath is device path displayed in container
 	ContainerPath string
+
+	// FileMode permission bits for the device.
+	FileMode os.FileMode
+
+	// UID is user ID in the container namespace
+	UID uint32
+
+	// GID is group ID in the container namespace
+	GID uint32
 }
 
 // Container is composed of a set of containers and a runtime environment.
@@ -680,6 +690,9 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err
 			storedDevices = append(storedDevices, ContainerDevice{
 				ID:            dev.DeviceID(),
 				ContainerPath: info.ContainerPath,
+				FileMode:      info.FileMode,
+				UID:           info.UID,
+				GID:           info.GID,
 			})
 		}
 		c.devices = filterDevices(sandbox, c, storedDevices)
@@ -749,6 +762,8 @@ func (c *Container) create() (err error) {
 		return
 	}
 
+	c.sandbox.PersistDevices()
+
 	process, err := c.sandbox.agent.createContainer(c.sandbox, c)
 	if err != nil {
 		return err

virtcontainers/device/api/interface.go

@@ -10,6 +10,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 var devLogger = logrus.WithField("subsystem", "device")
@@ -63,6 +64,9 @@ type Device interface {
 	Reference() uint
 	// Dereference removes one reference to device then returns final ref count
 	Dereference() uint
+
+	// Persist convert and return data in persist format
+	Dump() persistapi.DeviceState
 }
 
 // DeviceManager can be used to create a new device, this can be used as single

virtcontainers/device/drivers/block.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -142,5 +143,25 @@ func (device *BlockDevice) GetDeviceInfo() interface{} {
 	return device.BlockDrive
 }
 
+// Persist convert and return data in persist format
+func (device *BlockDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+
+	drive := device.BlockDrive
+	ds.BlockDrive = &persistapi.BlockDrive{
+		File:     drive.File,
+		Format:   drive.Format,
+		ID:       drive.ID,
+		Index:    drive.Index,
+		MmioAddr: drive.MmioAddr,
+		PCIAddr:  drive.PCIAddr,
+		SCSIAddr: drive.SCSIAddr,
+		NvdimmID: drive.NvdimmID,
+		VirtPath: drive.VirtPath,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/generic.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // GenericDevice refers to a device that is neither a VFIO device or block device.
@@ -127,3 +128,19 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error)
 		}
 	}
 }
+
+// Persist convert and return data in persist format
+func (device *GenericDevice) Dump() persistapi.DeviceState {
+	info := device.DeviceInfo
+	return persistapi.DeviceState{
+		ID:          device.ID,
+		Type:        string(device.DeviceType()),
+		RefCount:    device.RefCount,
+		AttachCount: device.AttachCount,
+
+		DevType:       info.DevType,
+		Major:         info.Major,
+		Minor:         info.Minor,
+		DriverOptions: info.DriverOptions,
+	}
+}

virtcontainers/device/drivers/vfio.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -129,6 +130,23 @@ func (device *VFIODevice) GetDeviceInfo() interface{} {
 	return device.VfioDevs
 }
 
+// Persist convert and return data in persist format
+func (device *VFIODevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+
+	devs := device.VfioDevs
+	for _, dev := range devs {
+		ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{
+			ID:       dev.ID,
+			Type:     string(dev.Type),
+			BDF:      dev.BDF,
+			SysfsDev: dev.SysfsDev,
+		})
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes
 

virtcontainers/device/drivers/vhost_user_blk.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -79,5 +80,18 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Persist convert and return data in persist format
+func (device *VhostUserBlkDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/vhost_user_net.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -79,5 +80,18 @@ func (device *VhostUserNetDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Persist convert and return data in persist format
+func (device *VhostUserNetDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/device/drivers/vhost_user_scsi.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -78,5 +79,18 @@ func (device *VhostUserSCSIDevice) GetDeviceInfo() interface{} {
 	return &device.VhostUserDeviceAttrs
 }
 
+// Persist convert and return data in persist format
+func (device *VhostUserSCSIDevice) Dump() persistapi.DeviceState {
+	ds := device.GenericDevice.Dump()
+	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+		DevID:      device.DevID,
+		SocketPath: device.SocketPath,
+		Type:       string(device.Type),
+		MacAddress: device.MacAddress,
+	}
+	return ds
+}
+
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes

virtcontainers/persist.go

@@ -10,6 +10,7 @@ import (
 
 	//"github.com/sirupsen/logrus"
 
+	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
@@ -28,14 +29,17 @@ func (s *Sandbox) PersistState() {
 		ss.GuestMemoryBlockSizeMB = s.state.GuestMemoryBlockSizeMB
 
 		for id, cont := range s.containers {
-			cs[id] = persistapi.ContainerState{
-				State: string(cont.state.State),
-				Rootfs: persistapi.RootfsState{
-					BlockDeviceID: cont.state.BlockDeviceID,
-					FsType:        cont.state.Fstype,
-				},
-				ShimPid: cont.state.Pid,
+			state := persistapi.ContainerState{}
+			if v, ok := cs[id]; ok {
+				state = v
+			}
+			state.State = string(cont.state.State)
+			state.Rootfs = persistapi.RootfsState{
+				BlockDeviceID: cont.state.BlockDeviceID,
+				FsType:        cont.state.Fstype,
 			}
+			state.ShimPid = cont.state.Pid
+			cs[id] = state
 		}
 		return nil
 	})
@@ -49,6 +53,41 @@ func (s *Sandbox) PersistHvState() {
 	})
 }
 
+func deviceToDeviceState(devices []api.Device) (dss []persistapi.DeviceState) {
+	for _, dev := range devices {
+		dss = append(dss, dev.Dump())
+	}
+	return
+}
+
+// PersistDevices register hook to save device informations
+func (s *Sandbox) PersistDevices() {
+	s.store.RegisterHook("devices", func(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) error {
+		ss.Devices = deviceToDeviceState(s.devManager.GetAllDevices())
+
+		for id, cont := range s.containers {
+			state := persistapi.ContainerState{}
+			if v, ok := cs[id]; ok {
+				state = v
+			}
+
+			state.DeviceMaps = nil
+			for _, dev := range cont.devices {
+				state.DeviceMaps = append(state.DeviceMaps, persistapi.DeviceMap{
+					ID:            dev.ID,
+					ContainerPath: dev.ContainerPath,
+					FileMode:      dev.FileMode,
+					UID:           dev.UID,
+					GID:           dev.GID,
+				})
+			}
+
+			cs[id] = state
+		}
+		return nil
+	})
+}
+
 // Restore will restore data from persist disk on disk
 func (s *Sandbox) Restore() error {
 	if err := s.store.Restore(s.id); err != nil {

virtcontainers/persist/api/device.go

@@ -23,13 +23,19 @@ type BlockDrive struct {
 	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
 	Index int
 
+	// MmioAddr is used to identify the slot at which the drive is attached (order?).
+	MmioAddr string
+
 	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
 	PCIAddr string
 
 	// SCSI Address of the block device, in case the device is attached using SCSI driver
 	// SCSI address is in the format SCSI-Id:LUN
 	SCSIAddr string
 
+	// NvdimmID is the nvdimm id inside the VM
+	NvdimmID string
+
 	// VirtPath at which the device appears inside the VM, outside of the container mount namespace
 	VirtPath string
 }

virtcontainers/sandbox.go

@@ -2009,6 +2009,7 @@ func (s *Sandbox) AddDevice(info config.DeviceInfo) (api.Device, error) {
 		return nil, err
 	}
 
+	s.PersistDevices()
 	return b, nil
 }